Turn off color for downstairs dump Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.27599978 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.020012908 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.019140784 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1189 1190 1191 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.34059111Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.406128851Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.407443919Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.407489202Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.40750595Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.407520858Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193} {"msg":"Crucible 59bc7765-e82c-4e97-99a9-0f330edffb41 has session id: 6046a79c-6a05-4ed7-819e-655690483c32","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.408529251Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"Upstairs opts: Upstairs UUID: 59bc7765-e82c-4e97-99a9-0f330edffb41, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.408594581Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"Crucible stats registered with UUID: 59bc7765-e82c-4e97-99a9-0f330edffb41","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.408617248Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.448536029Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.448584422Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193} {"msg":"59bc7765-e82c-4e97-99a9-0f330edffb41 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.448648353Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.448698156Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"client":"0","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.448786503Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"client":"0","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.448816678Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"client":"1","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.448904935Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"client":"1","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.448957907Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"client":"2","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.448989133Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"client":"2","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.449016739Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.449113235Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.44914879Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"ds_connection connected from Ok(127.0.0.1:34395)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.449221359Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"ds_connection connected from Ok(127.0.0.1:42640)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.449254494Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"ds_connection connected from Ok(127.0.0.1:34663)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.44928173Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.451047673Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"client":"0","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.451076528Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"client":"0","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.451092416Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"client":"1","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.451106544Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"client":"1","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.451152517Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"client":"2","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.451184542Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"client":"2","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.452136933Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"client":"1","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.452174108Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"client":"2","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.452195265Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"client":"0","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.452916649Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"client":"1","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.452947865Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"client":"1","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.452962613Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.452976601Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"client":"2","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.452991039Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"client":"2","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.453004477Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.453016835Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"client":"0","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.453030283Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"client":"0","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.453159094Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"":"downstairs","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.453178691Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"":"downstairs","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.453196469Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"":"downstairs","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.45325474Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"":"downstairs","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.453273588Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"":"downstairs","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.453287895Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"":"downstairs","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.453301483Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"":"downstairs","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.453315101Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"":"downstairs","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.453329149Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"":"downstairs","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.453342467Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"":"downstairs","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.453355845Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"":"downstairs","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.453369123Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"":"downstairs","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.453474468Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"":"downstairs","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.453499604Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.453524381Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"client":"0","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.453551957Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"client":"1","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.453567885Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"client":"2","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.453584072Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.45359698Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"59bc7765-e82c-4e97-99a9-0f330edffb41 is now active with session: 6046a79c-6a05-4ed7-819e-655690483c32","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.453612908Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"Set Active after no reconciliation","v":0Wait for a query_work_queue command to finish before sending IO ,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.453639074Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:44.453662521Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 88 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:30:48.534501821Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193} {"msg":"client request queue closed unexpectedly; is the program exiting?","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:30:48.534583909Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:30:48.534621383Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:30:48.534651399Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.534679485Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"6046a79c-6a05-4ed7-819e-655690483c32"} Begin loop 1 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.567210265Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.612904873Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.613620439Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.613655773Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.613670341Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.613685039Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195} {"msg":"Crucible 4e5a3d2d-b6b5-4e7e-a244-1fffd473dda4 has session id: e4dc66af-4632-4f48-b0de-2bc6d773947f","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.614064874Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"Upstairs opts: Upstairs UUID: 4e5a3d2d-b6b5-4e7e-a244-1fffd473dda4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.614111947Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"Crucible stats registered with UUID: 4e5a3d2d-b6b5-4e7e-a244-1fffd473dda4","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.614137263Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.645285605Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.64531927Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195} {"msg":"4e5a3d2d-b6b5-4e7e-a244-1fffd473dda4 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.645355145Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.645372782Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"client":"0","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.645401448Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"client":"0","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.645420115Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"client":"1","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.645441652Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"client":"1","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.64545863Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"client":"2","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.645475677Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"client":"2","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.645500544Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"":"io task","client":"2","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.645534829Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"":"io task","client":"1","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.645564724Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"":"io task","client":"0","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"ds_connection connected from Ok(127.0.0.1:49027)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.645599239Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"":"io task","client":"0","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"ds_connection connected from Ok(127.0.0.1:47312)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.645634394Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"":"io task","client":"2","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"ds_connection connected from Ok(127.0.0.1:43942)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.64566453Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"":"io task","client":"1","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.64634644Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"client":"2","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.646374956Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"client":"2","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.646390624Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"client":"0","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.646404762Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"client":"0","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.646581626Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"client":"1","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.646606582Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"client":"1","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.647141674Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"client":"0","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.64717146Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"client":"2","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.647186967Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"client":"1","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.64771592Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"client":"1","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.647746456Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"client":"1","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.647762663Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.648103804Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"client":"2","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.64813011Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"client":"2","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.648144947Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.648163925Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"client":"0","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.648178073Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"client":"0","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.648206449Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"":"downstairs","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.648220787Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"":"downstairs","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.648234944Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"":"downstairs","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.648248672Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"":"downstairs","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.64826576Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"":"downstairs","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.648280318Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"":"downstairs","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.648294136Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"":"downstairs","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.648307734Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"":"downstairs","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.648321682Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"":"downstairs","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.64833482Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"":"downstairs","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.648349148Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"":"downstairs","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.648362196Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"":"downstairs","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.648376734Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"":"downstairs","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:30:48.648390072Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.648412828Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"client":"0","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.648440464Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"client":"1","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.648465771Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"client":"2","":"downstairs","session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.648490897Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.648511784Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 4e5a3d2d-b6b5-4e7e-a244-1fffd473dda4 is now active with session: e4dc66af-4632-4f48-b0de-2bc6d773947f","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.648533771Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.648557597Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195,"session_id":"e4dc66af-4632-4f48-b0de-2bc6d773947f"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:48.648581924Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 503 len 20480 data: 2 2 2 2 2 02/30 Write block 95 len 40960 data: 2 2 2 2 2 2 2 2 2 2 03/30 Write block 118 len 24576 data: 2 2 2 2 2 2 04/30 Write block 158 len 40960 data: 2 2 2 2 2 2 2 2 2 2 05/30 Read block 258 len 20480 06/30 Write block 160 len 28672 data: 3 3 3 3 3 3 3 07/30 Write block 248 len 12288 data: 2 2 2 08/30 Write block 297 len 36864 data: 2 2 2 2 2 2 2 2 2 09/30 Flush 10/30 Write block 138 len 12288 data: 2 2 2 11/30 Flush 12/30 Write block 380 len 24576 data: 2 2 2 2 2 2 13/30 Read block 118 len 8192 14/30 Read block 485 len 40960 15/30 Write block 259 len 40960 data: 2 2 2 2 2 2 2 2 2 2 16/30 Write block 525 len 32768 data: 2 2 2 2 2 2 2 2 17/30 Write block 366 len 32768 data: 2 2 2 2 2 2 2 2 18/30 Read block 578 len 32768 19/30 Write block 414 len 32768 data: 2 2 2 2 2 2 2 2 20/30 Write block 282 len 16384 data: 2 2 2 2 21/30 Write block 446 len 40960 data: 2 2 2 2 2 2 2 2 2 2 22/30 Write block 102 len 8192 data: 3 3 23/30 Write block 448 len 32768 data: 3 3 3 3 3 3 3 3 24/30 Read block 71 len 8192 25/30 Read block 517 len 12288 26/30 Write block 461 len 20480 data: 2 2 2 2 2 27/30 Write block 372 len 8192 data: 3 3 28/30 Write block 67 len 4096 data: 2 29/30 Read block 291 len 16384 30/30 Read block 17 len 32768 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 6 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Read 10 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 19 18 18 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:30:48.667893514Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1195} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 1 1 1 1 1 1 T T T 4 080-099 2 2 2 2 2 2 F F F 5 100-119 2 2 2 2 2 2 T T T 6 120-139 2 2 2 3 3 3 F F F 7 140-159 2 2 2 3 3 3 F F F 8 160-179 2 2 2 2 2 2 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 2 2 2 2 2 2 T T T 13 260-279 1 1 1 1 1 1 T T T 14 280-299 2 2 2 2 2 2 T T T 15 300-319 2 2 2 2 2 2 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 T T T 19 380-399 1 1 1 1 1 1 T T T 20 400-419 1 1 1 1 1 1 T T T 21 420-439 1 1 1 1 1 1 T T T 22 440-459 1 1 1 1 1 1 T T T 23 460-479 1 1 1 1 1 1 T T T 24 480-499 1 1 1 1 1 1 F F F 25 500-519 2 2 2 2 2 2 F F F 26 520-539 1 1 1 1 1 1 T T T 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 3 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 3 3 3 4 4 4 F F F 4 080-099 2 2 2 2 2 2 F F F 5 100-119 3 3 3 4 4 4 F F F 6 120-139 2 2 2 3 3 3 F F F 7 140-159 2 2 2 3 3 3 F F F 8 160-179 2 2 2 2 2 2 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 3 3 3 4 4 4 F F F 13 260-279 3 3 3 4 4 4 F F F 14 280-299 3 3 3 4 4 4 F F F 15 300-319 2 2 2 2 2 2 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 3 3 3 4 4 4 F F F 19 380-399 3 3 3 4 4 4 F F F 20 400-419 3 3 3 4 4 4 F F F 21 420-439 3 3 3 4 4 4 F F F 22 440-459 3 3 3 4 4 4 F F F 23 460-479 3 3 3 4 4 4 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 2 2 2 2 2 2 F F F 26 520-539 3 3 3 4 4 4 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 4 Begin loop 2 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:52.987987939Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.034229247Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.034905319Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.034943633Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.034958631Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.034974639Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203} {"msg":"Crucible 166e0ad7-39f4-49ee-90db-a5f459ddf1eb has session id: 5af7b19e-dfbd-4471-9975-e32d0b7f30a0","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.035439341Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"Upstairs opts: Upstairs UUID: 166e0ad7-39f4-49ee-90db-a5f459ddf1eb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.035476495Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"Crucible stats registered with UUID: 166e0ad7-39f4-49ee-90db-a5f459ddf1eb","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.035492383Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.067115186Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.067169368Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203} {"msg":"166e0ad7-39f4-49ee-90db-a5f459ddf1eb active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.067185905Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.067202383Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"client":"0","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.067241247Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"client":"0","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.067259865Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"client":"1","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.067276952Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"client":"1","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.067299559Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"client":"2","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.067317306Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"client":"2","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.067333824Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"":"io task","client":"2","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.067351141Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"":"io task","client":"0","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.067368558Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"":"io task","client":"1","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:40327)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.067486121Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"":"io task","client":"2","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:43499)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.067535854Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"":"io task","client":"1","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:48922)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.067569219Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"":"io task","client":"0","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.068367313Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"client":"0","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.068398138Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"client":"0","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.068414266Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"client":"1","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.068436893Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"client":"1","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.068500393Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"client":"2","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.068529329Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"client":"2","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.06900479Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"client":"1","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.069037395Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"client":"0","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.06914185Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"client":"2","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.06961984Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"client":"0","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.069650016Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"client":"0","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.069667133Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.069679971Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"client":"1","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.069693549Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"client":"1","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.069707187Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.069720055Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"client":"2","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.069733753Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"client":"2","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 4, 2, 4, 3, 3, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.069747371Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"":"downstairs","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 3, 2, 3, 2, 2, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.069762029Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"":"downstairs","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.069776337Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"":"downstairs","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 4, 2, 4, 3, 3, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.069790115Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"":"downstairs","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 3, 2, 3, 2, 2, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.069803853Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"":"downstairs","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.06982152Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"":"downstairs","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 4, 2, 4, 3, 3, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.069835818Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"":"downstairs","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 3, 2, 3, 2, 2, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.069849676Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"":"downstairs","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.069863754Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"":"downstairs","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.069876842Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"":"downstairs","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:30:53.06989011Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"":"downstairs","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"Next flush: 5","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.069908498Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"":"downstairs","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.069921546Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"":"downstairs","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.069934884Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.069946932Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"client":"0","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.06996017Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"client":"1","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.069973298Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"client":"2","":"downstairs","session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.069986646Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.069998945Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"166e0ad7-39f4-49ee-90db-a5f459ddf1eb is now active with session: 5af7b19e-dfbd-4471-9975-e32d0b7f30a0","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.070012103Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.070024301Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203,"session_id":"5af7b19e-dfbd-4471-9975-e32d0b7f30a0"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:53.070038689Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 587 len 16384 data: 2 2 2 2 02/30 Write block 296 len 16384 data: 2 3 3 3 03/30 Read block 350 len 12288 04/30 Read block 153 len 4096 05/30 Write block 45 len 12288 data: 2 2 2 06/30 Read block 453 len 20480 07/30 Write block 571 len 20480 data: 2 2 2 2 2 08/30 Read block 426 len 12288 09/30 Write block 451 len 12288 data: 4 4 4 10/30 Read block 195 len 8192 11/30 Read block 535 len 8192 12/30 Write block 175 len 32768 data: 2 2 2 2 2 2 2 2 13/30 Write block 562 len 36864 data: 2 2 2 2 2 2 2 2 2 14/30 Read block 249 len 28672 15/30 Write block 159 len 28672 data: 3 4 4 4 4 4 4 16/30 Read block 512 len 8192 17/30 Read block 290 len 12288 18/30 Flush 19/30 Write block 238 len 32768 data: 2 2 2 2 2 2 2 2 20/30 Read block 241 len 28672 21/30 Write block 275 len 16384 data: 2 2 2 2 22/30 Write block 216 len 32768 data: 2 2 2 2 2 2 2 2 23/30 Read block 143 len 16384 24/30 Read block 8 len 16384 25/30 Write block 117 len 36864 data: 2 3 3 3 3 3 3 2 2 26/30 Read block 449 len 32768 27/30 Read block 85 len 12288 28/30 Read block 400 len 4096 29/30 Read block 187 len 36864 30/30 Write block 550 len 36864 data: 2 2 2 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 8 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:30:53.089716555Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1203} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 4 4 4 5 5 5 F F F 3 060-079 3 3 3 4 4 4 F F F 4 080-099 2 2 2 2 2 2 F F F 5 100-119 3 3 3 4 4 4 T T T 6 120-139 2 2 2 3 3 3 T T T 7 140-159 4 4 4 5 5 5 F F F 8 160-179 4 4 4 5 5 5 F F F 9 180-199 4 4 4 5 5 5 F F F 10 200-219 1 1 1 1 1 1 T T T 11 220-239 1 1 1 1 1 1 T T T 12 240-259 3 3 3 4 4 4 T T T 13 260-279 3 3 3 4 4 4 T T T 14 280-299 4 4 4 5 5 5 F F F 15 300-319 2 2 2 2 2 2 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 3 3 3 4 4 4 F F F 19 380-399 3 3 3 4 4 4 F F F 20 400-419 3 3 3 4 4 4 F F F 21 420-439 3 3 3 4 4 4 F F F 22 440-459 4 4 4 5 5 5 F F F 23 460-479 3 3 3 4 4 4 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 2 2 2 2 2 2 F F F 26 520-539 3 3 3 4 4 4 F F F 27 540-559 1 1 1 1 1 1 T T T 28 560-579 4 4 4 5 5 5 F F F 29 580-599 4 4 4 5 5 5 F F F Max gen: 4, Max flush: 5 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 4 4 4 5 5 5 F F F 3 060-079 3 3 3 4 4 4 F F F 4 080-099 2 2 2 2 2 2 F F F 5 100-119 5 5 5 6 6 6 F F F 6 120-139 5 5 5 6 6 6 F F F 7 140-159 4 4 4 5 5 5 F F F 8 160-179 4 4 4 5 5 5 F F F 9 180-199 4 4 4 5 5 5 F F F 10 200-219 5 5 5 6 6 6 F F F 11 220-239 5 5 5 6 6 6 F F F 12 240-259 5 5 5 6 6 6 F F F 13 260-279 5 5 5 6 6 6 F F F 14 280-299 4 4 4 5 5 5 F F F 15 300-319 2 2 2 2 2 2 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 3 3 3 4 4 4 F F F 19 380-399 3 3 3 4 4 4 F F F 20 400-419 3 3 3 4 4 4 F F F 21 420-439 3 3 3 4 4 4 F F F 22 440-459 4 4 4 5 5 5 F F F 23 460-479 3 3 3 4 4 4 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 2 2 2 2 2 2 F F F 26 520-539 3 3 3 4 4 4 F F F 27 540-559 5 5 5 6 6 6 F F F 28 560-579 4 4 4 5 5 5 F F F 29 580-599 4 4 4 5 5 5 F F F Max gen: 5, Max flush: 6 Begin loop 3 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.36386769Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:30:57.406540529Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.406904126Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.438767294Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.439323163Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.439366656Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.439388733Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.43940732Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211} {"msg":"Crucible 32569958-f077-48c1-a989-f0d4eaa9b9d6 has session id: 43a51950-877f-4e7f-9b51-7bab3992b25c","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.4398875Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"Upstairs opts: Upstairs UUID: 32569958-f077-48c1-a989-f0d4eaa9b9d6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.439929544Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"Crucible stats registered with UUID: 32569958-f077-48c1-a989-f0d4eaa9b9d6","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.439946821Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.470664556Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.470699421Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211} {"msg":"32569958-f077-48c1-a989-f0d4eaa9b9d6 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.470722678Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.470739595Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"client":"0","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.470767021Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"client":"0","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.470785049Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"client":"1","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.470802686Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"client":"1","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.470819584Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"client":"2","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.470836711Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"client":"2","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.470858048Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"":"io task","client":"0","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.470887494Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"":"io task","client":"2","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.470919389Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"":"io task","client":"1","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"ds_connection connected from Ok(127.0.0.1:32790)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.470950094Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"":"io task","client":"0","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"ds_connection connected from Ok(127.0.0.1:49815)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.470984029Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"":"io task","client":"1","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"ds_connection connected from Ok(127.0.0.1:62246)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.471062918Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"":"io task","client":"2","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.471654412Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"client":"1","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.471692216Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"client":"1","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.471796761Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"client":"2","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.471860912Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"client":"2","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.472038056Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"client":"0","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.47207873Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"client":"0","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.472258083Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"client":"1","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.47241862Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"client":"2","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.472767159Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"client":"0","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.472930675Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"client":"1","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.472953042Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"client":"1","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.47296774Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.473075054Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"client":"2","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.473096831Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"client":"2","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.473111109Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.473408626Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"client":"0","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.473432382Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"client":"0","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 5, 4, 2, 6, 6, 5, 5, 5, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.47344656Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"":"downstairs","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"[0]R generation[0..12]: [1, 1, 4, 3, 2, 5, 5, 4, 4, 4, 5, 5]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.473460548Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"":"downstairs","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.473475026Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"":"downstairs","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 5, 4, 2, 6, 6, 5, 5, 5, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.473488594Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"":"downstairs","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"[1]R generation[0..12]: [1, 1, 4, 3, 2, 5, 5, 4, 4, 4, 5, 5]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.473502062Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"":"downstairs","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.47351618Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"":"downstairs","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 5, 4, 2, 6, 6, 5, 5, 5, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.473529858Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"":"downstairs","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"[2]R generation[0..12]: [1, 1, 4, 3, 2, 5, 5, 4, 4, 4, 5, 5]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.473547185Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"":"downstairs","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.473561653Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"":"downstairs","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.473575071Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"":"downstairs","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.473588289Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"":"downstairs","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"Next flush: 7","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.473601357Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"":"downstairs","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.473614265Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"":"downstairs","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.473627463Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"Transition from WaitQuorum to Active","vWait for a query_work_queue command to finish before sending IO ":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.47365135Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"client":"0","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.473664968Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"client":"1","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.473678576Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"client":"2","":"downstairs","session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.473692274Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.473704712Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"32569958-f077-48c1-a989-f0d4eaa9b9d6 is now active with session: 43a51950-877f-4e7f-9b51-7bab3992b25c","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.47371766Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.473729809Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211,"session_id":"43a51950-877f-4e7f-9b51-7bab3992b25c"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:30:57.473742317Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 182 len 36864 data: 3 2 2 2 2 2 2 2 2 02/30 Write block 293 len 8192 data: 2 2 03/30 Write block 139 len 36864 data: 3 3 2 2 2 2 2 2 2 04/30 Read block 193 len 4096 05/30 Write block 391 len 28672 data: 2 2 2 2 2 2 2 06/30 Read block 386 len 24576 07/30 Write block 135 len 20480 data: 2 2 2 3 4 08/30 Read block 100 len 20480 09/30 Write block 357 len 36864 data: 2 2 2 2 2 2 2 2 2 10/30 Write block 591 len 20480 data: 2 2 2 2 2 11/30 Write block 521 len 36864 data: 2 2 2 2 3 3 3 3 3 12/30 Flush 13/30 Flush 14/30 Flush 15/30 Write block 376 len 24576 data: 2 2 2 2 3 3 16/30 Write block 62 len 28672 data: 2 2 2 2 2 3 2 17/30 Write block 199 len 40960 data: 2 2 2 2 2 2 2 2 2 2 18/30 Write block 90 len 40960 data: 2 2 2 2 2 3 3 3 3 3 19/30 Read block 20 len 24576 20/30 Read block 384 len 24576 21/30 Read block 299 len 8192 22/30 Write block 320 len 4096 data: 2 23/30 Read block 113 len 40960 24/30 Read block 219 len 20480 25/30 Write block 48 len 20480 data: 2 2 2 2 2 26/30 Write block 49 len 16384 data: 3 3 3 3 27/30 Write block 522 len 32768 data: 3 3 3 4 4 4 4 4 28/30 Write block 386 len 24576 data: 2 2 2 2 2 3 29/30 Write block 85 len 20480 data: 2 2 2 2 2 30/30 Write block 398 len 36864 data: 2 2 2 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 6 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 5 Sent Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 1 1 4 Done 14 15 15 44 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:30:57.492798374Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1211} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 4 4 4 5 5 5 T T T 3 060-079 3 3 3 4 4 4 T T T 4 080-099 2 2 2 2 2 2 T T T 5 100-119 5 5 5 6 6 6 F F F 6 120-139 6 6 6 7 7 7 F F F 7 140-159 6 6 6 7 7 7 F F F 8 160-179 4 4 4 5 5 5 F F F 9 180-199 6 6 6 7 7 7 T T T 10 200-219 5 5 5 6 6 6 T T T 11 220-239 5 5 5 6 6 6 F F F 12 240-259 5 5 5 6 6 6 F F F 13 260-279 5 5 5 6 6 6 F F F 14 280-299 6 6 6 7 7 7 F F F 15 300-319 2 2 2 2 2 2 F F F 16 320-339 1 1 1 1 1 1 T T T 17 340-359 6 6 6 7 7 7 F F F 18 360-379 6 6 6 7 7 7 T T T 19 380-399 6 6 6 7 7 7 T T T 20 400-419 3 3 3 4 4 4 T T T 21 420-439 3 3 3 4 4 4 F F F 22 440-459 4 4 4 5 5 5 F F F 23 460-479 3 3 3 4 4 4 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 2 2 2 2 2 2 F F F 26 520-539 6 6 6 7 7 7 T T T 27 540-559 5 5 5 6 6 6 F F F 28 560-579 4 4 4 5 5 5 F F F 29 580-599 6 6 6 7 7 7 F F F Max gen: 6, Max flush: 7 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 7 7 7 8 8 8 F F F 3 060-079 7 7 7 8 8 8 F F F 4 080-099 7 7 7 8 8 8 F F F 5 100-119 5 5 5 6 6 6 F F F 6 120-139 6 6 6 7 7 7 F F F 7 140-159 6 6 6 7 7 7 F F F 8 160-179 4 4 4 5 5 5 F F F 9 180-199 7 7 7 8 8 8 F F F 10 200-219 7 7 7 8 8 8 F F F 11 220-239 5 5 5 6 6 6 F F F 12 240-259 5 5 5 6 6 6 F F F 13 260-279 5 5 5 6 6 6 F F F 14 280-299 6 6 6 7 7 7 F F F 15 300-319 2 2 2 2 2 2 F F F 16 320-339 7 7 7 8 8 8 F F F 17 340-359 6 6 6 7 7 7 F F F 18 360-379 7 7 7 8 8 8 F F F 19 380-399 7 7 7 8 8 8 F F F 20 400-419 7 7 7 8 8 8 F F F 21 420-439 3 3 3 4 4 4 F F F 22 440-459 4 4 4 5 5 5 F F F 23 460-479 3 3 3 4 4 4 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 2 2 2 2 2 2 F F F 26 520-539 7 7 7 8 8 8 F F F 27 540-559 5 5 5 6 6 6 F F F 28 560-579 4 4 4 5 5 5 F F F 29 580-599 6 6 6 7 7 7 F F F Max gen: 7, Max flush: 8 Begin loop 4 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.813772251Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.856976292Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.857668821Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.857704586Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.857724103Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.857739991Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219} {"msg":"Crucible 789efae9-a285-4700-aae0-a43003559530 has session id: 18fd6a04-cd58-4490-9ad8-c4eb94a2786d","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.858238618Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"Upstairs opts: Upstairs UUID: 789efae9-a285-4700-aae0-a43003559530, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.858282882Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"Crucible stats registered with UUID: 789efae9-a285-4700-aae0-a43003559530","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.858312407Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.889816827Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.889849272Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219} {"msg":"789efae9-a285-4700-aae0-a43003559530 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.889899275Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.889919642Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"client":"0","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.889946648Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"client":"0","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.889964776Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"client":"1","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.889981703Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"client":"1","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.889998271Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"client":"2","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.890015378Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"client":"2","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.890044984Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"":"io task","client":"2","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.890076839Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"":"io task","client":"0","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.890113764Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"":"io task","client":"1","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"ds_connection connected from Ok(127.0.0.1:40299)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.890178104Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"":"io task","client":"1","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"ds_connection connected from Ok(127.0.0.1:36552)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.890231867Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"":"io task","client":"0","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"ds_connection connected from Ok(127.0.0.1:61928)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.890272461Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"":"io task","client":"2","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.890936624Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"client":"2","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.890977968Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"client":"2","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.891009123Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"client":"0","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.891040409Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"client":"0","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.891072374Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"client":"1","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.891093321Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"client":"1","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.891498382Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"client":"2","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.891594058Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"client":"1","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.891615175Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"client":"0","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.89212354Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"client":"2","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.892151636Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"client":"2","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.892170094Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.892238254Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"client":"1","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.892258461Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"client":"1","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.892272139Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.892284607Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"client":"0","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.892298105Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"client":"0","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 8, 8, 8, 6, 7, 7, 5, 8, 8, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.892312003Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"":"downstairs","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"[0]R generation[0..12]: [1, 1, 7, 7, 7, 5, 6, 6, 4, 7, 7, 5]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.892325811Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"":"downstairs","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.892340099Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"":"downstairs","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 8, 8, 8, 6, 7, 7, 5, 8, 8, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.892353867Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"":"downstairs","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"[1]R generation[0..12]: [1, 1, 7, 7, 7, 5, 6, 6, 4, 7, 7, 5]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.892367525Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"":"downstairs","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.892381783Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"":"downstairs","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 8, 8, 8, 6, 7, 7, 5, 8, 8, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.892395261Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"":"downstairs","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"[2]R generation[0..12]: [1, 1, 7, 7, 7, 5, 6, 6, 4, 7, 7, 5]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.892409029Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"":"downstairs","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.892425566Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"":"downstairs","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"timeWait for a query_work_queue command to finish before sending IO ":"2025-01-03T08:31:01.892439014Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"":"downstairs","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.892523992Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"":"downstairs","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"Next flush: 9","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.89254099Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"":"downstairs","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.892554417Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"":"downstairs","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.892567886Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.892580254Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"client":"0","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.892594132Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"client":"1","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.89260786Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"client":"2","":"downstairs","session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.892621678Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.892634166Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"789efae9-a285-4700-aae0-a43003559530 is now active with session: 18fd6a04-cd58-4490-9ad8-c4eb94a2786d","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.892647064Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.892659332Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219,"session_id":"18fd6a04-cd58-4490-9ad8-c4eb94a2786d"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:01.89267167Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 190 len 12288 data: 3 2 2 03/30 Write block 10 len 20480 data: 2 2 2 2 2 04/30 Read block 136 len 32768 05/30 Read block 261 len 28672 06/30 Write block 419 len 28672 data: 3 3 3 2 2 2 2 07/30 Write block 134 len 36864 data: 2 3 3 3 4 5 4 3 3 08/30 Read block 440 len 40960 09/30 Read block 146 len 40960 10/30 Write block 414 len 32768 data: 3 3 3 3 3 4 4 4 11/30 Read block 42 len 16384 12/30 Write block 129 len 16384 data: 2 2 2 2 13/30 Write block 584 len 12288 data: 2 2 2 14/30 Read block 230 len 36864 15/30 Read block 40 len 36864 16/30 Read block 1 len 20480 17/30 Read block 101 len 20480 18/30 Read block 46 len 4096 19/30 Read block 576 len 32768 20/30 Write block 358 len 36864 data: 3 3 3 3 3 3 3 3 3 21/30 Read block 280 len 12288 22/30 Write block 579 len 24576 data: 2 2 2 2 2 3 23/30 Read block 577 len 20480 24/30 Flush 25/30 Write block 412 len 32768 data: 2 2 4 4 4 4 4 5 26/30 Read block 45 len 8192 27/30 Read block 406 len 32768 28/30 Write block 169 len 16384 data: 2 2 2 2 29/30 Read block 454 len 32768 30/30 Read block 467 len 36864 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 8 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 9 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 5 6 6 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:31:01.913383936Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1219} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 10 10 10 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 7 7 7 8 8 8 F F F 3 060-079 7 7 7 8 8 8 F F F 4 080-099 7 7 7 8 8 8 F F F 5 100-119 5 5 5 6 6 6 F F F 6 120-139 8 8 8 10 10 10 F F F 7 140-159 8 8 8 10 10 10 F F F 8 160-179 4 4 4 5 5 5 T T T 9 180-199 8 8 8 10 10 10 F F F 10 200-219 7 7 7 8 8 8 F F F 11 220-239 5 5 5 6 6 6 F F F 12 240-259 5 5 5 6 6 6 F F F 13 260-279 5 5 5 6 6 6 F F F 14 280-299 6 6 6 7 7 7 F F F 15 300-319 2 2 2 2 2 2 F F F 16 320-339 7 7 7 8 8 8 F F F 17 340-359 8 8 8 10 10 10 F F F 18 360-379 8 8 8 10 10 10 F F F 19 380-399 7 7 7 8 8 8 F F F 20 400-419 8 8 8 10 10 10 T T T 21 420-439 8 8 8 10 10 10 F F F 22 440-459 4 4 4 5 5 5 F F F 23 460-479 3 3 3 4 4 4 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 2 2 2 2 2 2 F F F 26 520-539 7 7 7 8 8 8 F F F 27 540-559 5 5 5 6 6 6 F F F 28 560-579 8 8 8 10 10 10 F F F 29 580-599 8 8 8 10 10 10 F F F Max gen: 8, Max flush: 10 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 10 10 10 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 7 7 7 8 8 8 F F F 3 060-079 7 7 7 8 8 8 F F F 4 080-099 7 7 7 8 8 8 F F F 5 100-119 5 5 5 6 6 6 F F F 6 120-139 8 8 8 10 10 10 F F F 7 140-159 8 8 8 10 10 10 F F F 8 160-179 9 9 9 11 11 11 F F F 9 180-199 8 8 8 10 10 10 F F F 10 200-219 7 7 7 8 8 8 F F F 11 220-239 5 5 5 6 6 6 F F F 12 240-259 5 5 5 6 6 6 F F F 13 260-279 5 5 5 6 6 6 F F F 14 280-299 6 6 6 7 7 7 F F F 15 300-319 2 2 2 2 2 2 F F F 16 320-339 7 7 7 8 8 8 F F F 17 340-359 8 8 8 10 10 10 F F F 18 360-379 8 8 8 10 10 10 F F F 19 380-399 7 7 7 8 8 8 F F F 20 400-419 9 9 9 11 11 11 F F F 21 420-439 8 8 8 10 10 10 F F F 22 440-459 4 4 4 5 5 5 F F F 23 460-479 3 3 3 4 4 4 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 2 2 2 2 2 2 F F F 26 520-539 7 7 7 8 8 8 F F F 27 540-559 5 5 5 6 6 6 F F F 28 560-579 8 8 8 10 10 10 F F F 29 580-599 8 8 8 10 10 10 F F F Max gen: 9, Max flush: 11 Begin loop 5 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.169615376Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.212679838Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.213383536Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.213427009Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.213445287Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.213463764Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227} {"msg":"Crucible ab50767a-e3a3-4795-86a8-61bf35d87524 has session id: 0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.213972849Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"Upstairs opts: Upstairs UUID: ab50767a-e3a3-4795-86a8-61bf35d87524, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.21403839Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"Crucible stats registered with UUID: ab50767a-e3a3-4795-86a8-61bf35d87524","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.214069426Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.244423663Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.244468777Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227} {"msg":"ab50767a-e3a3-4795-86a8-61bf35d87524 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.244506651Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.244538157Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"client":"0","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.24458572Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"client":"0","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.244624284Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"client":"1","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.244660449Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"client":"1","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.244696034Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"client":"2","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.244733208Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"client":"2","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.244755455Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"":"io task","client":"2","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.244774352Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"":"io task","client":"0","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.244808457Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"":"io task","client":"1","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:37818)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.244843632Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"":"io task","client":"0","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:44580)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.244874597Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"":"io task","client":"2","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:34857)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.244906443Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"":"io task","client":"1","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.245624768Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"client":"0","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.245666592Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"client":"0","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.245693728Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"client":"2","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.245794413Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"client":"2","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.245820889Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"client":"1","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.245835767Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"client":"1","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.246273933Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"client":"0","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.246368969Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"client":"1","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.246480953Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"client":"2","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.246883434Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"client":"0","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.24691133Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"client":"0","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.246931987Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.246957124Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"client":"1","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.24698084Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"client":"1","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.247001397Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.247021354Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"client":"2","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.247039741Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"client":"2","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"[0]R flush_numbers[0..12]: [10, 1, 8, 8, 8, 6, 10, 10, 11, 10, 8, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.247054479Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"":"downstairs","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"[0]R generation[0..12]: [8, 1, 7, 7, 7, 5, 8, 8, 9, 8, 7, 5]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.247068447Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"":"downstairs","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.247082535Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"":"downstairs","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"[1]R flush_numbers[0..12]: [10, 1, 8, 8, 8, 6, 10, 10, 11, 10, 8, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.247096473Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"":"downstairs","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"[1]R generation[0..12]: [8, 1, 7, 7, 7, 5, 8, 8, 9, 8, 7, 5]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.247110151Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"":"downstairs","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.247124339Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"":"downstairs","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"[2]R flush_numbers[0..12]: [10, 1, 8, 8, 8, 6, 10, 10, 11, 10, 8, 6]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.247137837Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"":"downstairs","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"[2]R generation[0..12]: [8, 1, 7, 7, 7, 5, 8, 8, 9, 8, 7, 5]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.247151255Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"":"downstairs","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.247165353Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"":"downstairs","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.247178471Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"":"downstairs","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.247191869Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"":"downstairs","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.247212556Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"":"downstairs","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.247226014Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"":"downstairs","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.247241022Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.24725353Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"client":"0","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.247266888Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"client":"1","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.247280146Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"client":"2","":"downstairs","session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.247293474Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.247305893Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ab50767a-e3a3-4795-86a8-61bf35d87524 is now active with session: 0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.2473216Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.247334978Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227,"session_id":"0d6d4ecf-169f-4b0b-b470-3ee509f2cd2a"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:06.247347647Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 391 len 16384 data: 4 3 3 3 02/30 Write block 83 len 12288 data: 2 2 3 03/30 Write block 543 len 40960 data: 2 2 2 2 2 2 2 3 3 3 04/30 Read block 472 len 16384 05/30 Write block 223 len 8192 data: 3 2 06/30 Write block 525 len 16384 data: 5 5 5 5 07/30 Write block 251 len 40960 data: 2 2 2 2 2 2 2 2 3 3 08/30 Flush 09/30 Write block 115 len 40960 data: 2 2 3 4 4 4 4 4 4 3 10/30 Read block 93 len 24576 11/30 Read block 265 len 12288 12/30 Read block 233 len 4096 13/30 Flush 14/30 Write block 211 len 16384 data: 2 2 2 2 15/30 Read block 576 len 36864 16/30 Read block 517 len 16384 17/30 Write block 158 len 40960 data: 3 4 5 5 5 5 5 5 4 3 18/30 Read block 565 len 36864 19/30 Read block 4 len 24576 20/30 Read block 347 len 20480 21/30 Read block 398 len 12288 22/30 Read block 118 len 40960 23/30 Write block 521 len 28672 data: 3 4 4 4 6 6 6 24/30 Read block 357 len 32768 25/30 Write block 247 len 32768 data: 2 3 3 3 3 3 3 3 26/30 Flush 27/30 Write block 502 len 36864 data: 2 3 3 3 3 3 2 2 2 28/30 Write block 119 len 20480 data: 5 5 5 5 5 29/30 Write block 58 len 16384 data: 2 2 2 2 30/30 Read block 581 len 20480 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 9 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 4 Sent Done Done false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 2 4 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:31:06.268086278Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1227} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 10 10 10 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 7 7 7 8 8 8 T T T 3 060-079 7 7 7 8 8 8 T T T 4 080-099 10 10 10 12 12 12 F F F 5 100-119 10 10 10 13 13 13 T T T 6 120-139 10 10 10 13 13 13 T T T 7 140-159 10 10 10 14 14 14 F F F 8 160-179 10 10 10 14 14 14 F F F 9 180-199 8 8 8 10 10 10 F F F 10 200-219 10 10 10 14 14 14 F F F 11 220-239 10 10 10 12 12 12 F F F 12 240-259 10 10 10 14 14 14 F F F 13 260-279 10 10 10 12 12 12 F F F 14 280-299 6 6 6 7 7 7 F F F 15 300-319 2 2 2 2 2 2 F F F 16 320-339 7 7 7 8 8 8 F F F 17 340-359 8 8 8 10 10 10 F F F 18 360-379 8 8 8 10 10 10 F F F 19 380-399 10 10 10 12 12 12 F F F 20 400-419 9 9 9 11 11 11 F F F 21 420-439 8 8 8 10 10 10 F F F 22 440-459 4 4 4 5 5 5 F F F 23 460-479 3 3 3 4 4 4 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 2 2 2 2 2 2 T T T 26 520-539 10 10 10 14 14 14 F F F 27 540-559 10 10 10 12 12 12 F F F 28 560-579 8 8 8 10 10 10 F F F 29 580-599 8 8 8 10 10 10 F F F Max gen: 10, Max flush: 14 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 10 10 10 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 11 11 11 15 15 15 F F F 3 060-079 11 11 11 15 15 15 F F F 4 080-099 10 10 10 12 12 12 F F F 5 100-119 11 11 11 15 15 15 F F F 6 120-139 11 11 11 15 15 15 F F F 7 140-159 10 10 10 14 14 14 F F F 8 160-179 10 10 10 14 14 14 F F F 9 180-199 8 8 8 10 10 10 F F F 10 200-219 10 10 10 14 14 14 F F F 11 220-239 10 10 10 12 12 12 F F F 12 240-259 10 10 10 14 14 14 F F F 13 260-279 10 10 10 12 12 12 F F F 14 280-299 6 6 6 7 7 7 F F F 15 300-319 2 2 2 2 2 2 F F F 16 320-339 7 7 7 8 8 8 F F F 17 340-359 8 8 8 10 10 10 F F F 18 360-379 8 8 8 10 10 10 F F F 19 380-399 10 10 10 12 12 12 F F F 20 400-419 9 9 9 11 11 11 F F F 21 420-439 8 8 8 10 10 10 F F F 22 440-459 4 4 4 5 5 5 F F F 23 460-479 3 3 3 4 4 4 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 11 11 11 15 15 15 F F F 26 520-539 10 10 10 14 14 14 F F F 27 540-559 10 10 10 12 12 12 F F F 28 560-579 8 8 8 10 10 10 F F F 29 580-599 8 8 8 10 10 10 F F F Max gen: 11, Max flush: 15 Begin loop 6 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.5354771Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.579396657Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.579981382Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.580018666Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.580033324Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.580047802Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235} {"msg":"Crucible 3d2db093-7d7f-4240-b538-2f66c0ea6d14 has session id: 25ec8fd8-ed02-4357-8209-9aba9bc65cff","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.580582644Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"Upstairs opts: Upstairs UUID: 3d2db093-7d7f-4240-b538-2f66c0ea6d14, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.580616169Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"Crucible stats registered with UUID: 3d2db093-7d7f-4240-b538-2f66c0ea6d14","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.580631416Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.61101925Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.611060453Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235} {"msg":"3d2db093-7d7f-4240-b538-2f66c0ea6d14 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.611091799Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.611112766Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"client":"0","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.611156879Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"client":"0","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.611199453Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"client":"1","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.611237368Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"client":"1","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.611270523Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"client":"2","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.611308227Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"client":"2","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.611342672Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.611378437Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"":"io task","client":"0","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.611527115Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"":"io task","client":"1","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"ds_connection connected from Ok(127.0.0.1:33449)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.611569369Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"ds_connection connected from Ok(127.0.0.1:62371)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.611604104Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"":"io task","client":"1","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"ds_connection connected from Ok(127.0.0.1:57449)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.61163178Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"":"io task","client":"0","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.612197587Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"client":"0","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.612226393Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"client":"0","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.612241981Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"client":"2","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.612256709Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"client":"2","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.612271117Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"client":"1","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.612285585Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"client":"1","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.612930191Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"client":"1","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.612959606Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"client":"2","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.612975504Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"client":"0","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.61354843Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"client":"1","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.613578206Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"client":"1","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.613594303Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.613607172Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"client":"2","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.6136208Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"client":"2","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.613634837Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.613647416Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"client":"0","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.613660894Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"client":"0","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"[0]R flush_numbers[0..12]: [10, 1, 15, 15, 12, 15, 15, 14, 14, 10, 14, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.613674732Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"":"downstairs","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"[0]R generation[0..12]: [8, 1, 11, 11, 10, 11, 11, 10, 10, 8, 10, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.61368872Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"":"downstairs","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.613703048Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"":"downstairs","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"[1]R flush_numbers[0..12]: [10, 1, 15, 15, 12, 15, 15, 14, 14, 10, 14, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.613720675Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"":"downstairs","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"[1]R generation[0..12]: [8, 1, 11, 11, 10, 11, 11, 10, 10, 8, 10, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.613735003Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"":"downstairs","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.613749271Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"":"downstairs","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"[2]R flush_numbers[0..12]: [10, 1, 15, 15, 12, 15, 15, 14, 14, 10, 14, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.613763019Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"":"downstairs","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"[2]R generation[0..12]: [8, 1, 11, 11, 10, 11, 11, 10, 10, 8, 10, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.613776717Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"":"downstairs","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.613790715Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"":"downstairs","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.613803903Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5Wait for a query_work_queue command to finish before sending IO ","pid":1235,"":"downstairs","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.613828989Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"":"downstairs","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"Next flush: 16","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.613842117Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"":"downstairs","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.613855365Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"":"downstairs","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.613868803Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.613880892Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"client":"0","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.61389415Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"client":"1","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.613907568Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"client":"2","":"downstairs","session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.613921066Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.613935494Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"3d2db093-7d7f-4240-b538-2f66c0ea6d14 is now active with session: 25ec8fd8-ed02-4357-8209-9aba9bc65cff","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.613948802Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.61396103Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235,"session_id":"25ec8fd8-ed02-4357-8209-9aba9bc65cff"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:10.613973448Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 56 len 36864 data: 2 2 3 3 3 3 3 3 3 03/30 Write block 124 len 28672 data: 4 3 2 2 2 3 3 04/30 Read block 450 len 40960 05/30 Write block 52 len 12288 data: 4 2 2 06/30 Read block 307 len 12288 07/30 Flush 08/30 Write block 351 len 40960 data: 2 2 2 2 2 2 3 4 4 4 09/30 Read block 286 len 36864 10/30 Write block 214 len 8192 data: 3 2 11/30 Read block 424 len 28672 12/30 Read block 81 len 4096 13/30 Flush 14/30 Write block 57 len 32768 data: 3 4 4 4 4 4 4 4 15/30 Read block 366 len 24576 16/30 Write block 454 len 28672 data: 4 4 2 2 2 2 2 17/30 Read block 447 len 12288 18/30 Flush 19/30 Write block 95 len 20480 data: 4 4 4 4 4 20/30 Read block 513 len 28672 21/30 Read block 155 len 12288 22/30 Write block 19 len 36864 data: 2 2 2 2 2 2 2 2 2 23/30 Write block 313 len 24576 data: 2 2 2 2 2 2 24/30 Read block 181 len 40960 25/30 Write block 551 len 4096 data: 4 26/30 Flush 27/30 Write block 425 len 20480 data: 3 2 2 2 2 28/30 Read block 140 len 40960 29/30 Read block 565 len 24576 30/30 Write block 240 len 32768 data: 3 3 3 3 3 3 2 3 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:31:10.633856675Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1235} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 20 20 20 F F F 1 020-039 12 12 12 20 20 20 F F F 2 040-059 12 12 12 19 19 19 F F F 3 060-079 12 12 12 19 19 19 F F F 4 080-099 12 12 12 20 20 20 F F F 5 100-119 11 11 11 15 15 15 F F F 6 120-139 12 12 12 17 17 17 F F F 7 140-159 10 10 10 14 14 14 F F F 8 160-179 10 10 10 14 14 14 F F F 9 180-199 8 8 8 10 10 10 F F F 10 200-219 12 12 12 18 18 18 F F F 11 220-239 10 10 10 12 12 12 F F F 12 240-259 10 10 10 14 14 14 T T T 13 260-279 10 10 10 12 12 12 F F F 14 280-299 6 6 6 7 7 7 F F F 15 300-319 12 12 12 20 20 20 F F F 16 320-339 7 7 7 8 8 8 F F F 17 340-359 12 12 12 18 18 18 F F F 18 360-379 12 12 12 18 18 18 F F F 19 380-399 10 10 10 12 12 12 F F F 20 400-419 9 9 9 11 11 11 F F F 21 420-439 8 8 8 10 10 10 T T T 22 440-459 12 12 12 19 19 19 F F F 23 460-479 12 12 12 19 19 19 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 11 11 11 15 15 15 F F F 26 520-539 10 10 10 14 14 14 F F F 27 540-559 12 12 12 20 20 20 F F F 28 560-579 8 8 8 10 10 10 F F F 29 580-599 8 8 8 10 10 10 F F F Max gen: 12, Max flush: 20 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 20 20 20 F F F 1 020-039 12 12 12 20 20 20 F F F 2 040-059 12 12 12 19 19 19 F F F 3 060-079 12 12 12 19 19 19 F F F 4 080-099 12 12 12 20 20 20 F F F 5 100-119 11 11 11 15 15 15 F F F 6 120-139 12 12 12 17 17 17 F F F 7 140-159 10 10 10 14 14 14 F F F 8 160-179 10 10 10 14 14 14 F F F 9 180-199 8 8 8 10 10 10 F F F 10 200-219 12 12 12 18 18 18 F F F 11 220-239 10 10 10 12 12 12 F F F 12 240-259 13 13 13 21 21 21 F F F 13 260-279 10 10 10 12 12 12 F F F 14 280-299 6 6 6 7 7 7 F F F 15 300-319 12 12 12 20 20 20 F F F 16 320-339 7 7 7 8 8 8 F F F 17 340-359 12 12 12 18 18 18 F F F 18 360-379 12 12 12 18 18 18 F F F 19 380-399 10 10 10 12 12 12 F F F 20 400-419 9 9 9 11 11 11 F F F 21 420-439 13 13 13 21 21 21 F F F 22 440-459 12 12 12 19 19 19 F F F 23 460-479 12 12 12 19 19 19 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 11 11 11 15 15 15 F F F 26 520-539 10 10 10 14 14 14 F F F 27 540-559 12 12 12 20 20 20 F F F 28 560-579 8 8 8 10 10 10 F F F 29 580-599 8 8 8 10 10 10 F F F Max gen: 13, Max flush: 21 Begin loop 7 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.880587612Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.924332295Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.92491826Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.924963613Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.924996898Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.925024824Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243} {"msg":"Crucible be5ea180-739d-4b54-9977-c576d08e36bb has session id: 84515fd3-e054-4a24-954a-51786bfb4e42","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.92538972Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"Upstairs opts: Upstairs UUID: be5ea180-739d-4b54-9977-c576d08e36bb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.925432954Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"Crucible stats registered with UUID: be5ea180-739d-4b54-9977-c576d08e36bb","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.925457301Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.956233697Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.956266672Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243} {"msg":"be5ea180-739d-4b54-9977-c576d08e36bb active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.956317755Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.956338222Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"client":"0","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.956366748Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"client":"0","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.956385575Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"client":"1","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.956404212Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"client":"1","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.956422519Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"client":"2","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.95655247Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"client":"2","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.956598934Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"":"io task","client":"2","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.956642287Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"":"io task","client":"0","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.956681631Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"":"io task","client":"1","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"ds_connection connected from Ok(127.0.0.1:53807)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.956823591Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"":"io task","client":"0","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"ds_connection connected from Ok(127.0.0.1:49720)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.956854406Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"":"io task","client":"2","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"ds_connection connected from Ok(127.0.0.1:57311)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.956871494Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"":"io task","client":"1","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.957430972Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"client":"0","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.957458568Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"client":"0","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.957473746Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"client":"2","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.957487944Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"client":"2","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.957567132Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"client":"1","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.957586629Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"client":"1","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.958058481Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"client":"0","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.958100304Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"client":"2","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.958213518Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"client":"1","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.958622238Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"client":"2","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.958654363Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"client":"2","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.958670741Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.958709945Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"client":"0","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.958727103Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"client":"0","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.958740461Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.958795223Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"client":"1","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.95881424Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"client":"1","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"[0]R flush_numbers[0..12]: [20, 20, 19, 19, 20, 15, 17, 14, 14, 10, 18, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.958828388Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"":"downstairs","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"[0]R generation[0..12]: [12, 12, 12, 12, 12, 11, 12, 10, 10, 8, 12, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.958842396Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"":"downstairs","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.958856614Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"":"downstairs","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"[1]R flush_numbers[0..12]: [20, 20, 19, 19, 20, 15, 17, 14, 14, 10, 18, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.958870202Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"":"downstairs","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"[1]R generation[0..12]: [12, 12, 12, 12, 12, 11, 12, 10, 10, 8, 12, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.95888381Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"":"downstairs","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.958897898Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"":"downstairs","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"[2]R flush_numbers[0..12]: [20, 20, 19, 19, 20, 15, 17, 14, 14, 10, 18, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.958911616Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"":"downstairs","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"[2]R generation[0..12]: [12, 12, 12, 12, 12, 11, 12, 10, 10, 8, 12, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.958925174Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"":"downstairs","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.958943181Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"":"downstairs","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.958956759Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"":"downstairs","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.958969977Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"":"downstairs","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"Next flush: 22","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.958982906Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"":"downstairs","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"All extents match","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2025-01-03T08:31:14.959003723Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"":"downstairs","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.95901835Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.959041107Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"client":"0","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.959066723Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"client":"1","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.95909065Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"client":"2","":"downstairs","session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.959116596Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.959142902Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"be5ea180-739d-4b54-9977-c576d08e36bb is now active with session: 84515fd3-e054-4a24-954a-51786bfb4e42","v":0,"name":"crucible","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 level":30,"time":"2025-01-03T08:31:14.959173618Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.959201264Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243,"session_id":"84515fd3-e054-4a24-954a-51786bfb4e42"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:14.959222141Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 252 len 24576 data: 4 4 4 3 3 3 02/30 Write block 339 len 16384 data: 2 2 2 2 03/30 Read block 433 len 40960 04/30 Read block 259 len 8192 05/30 Write block 62 len 4096 data: 5 06/30 Write block 576 len 32768 data: 2 2 2 3 3 3 3 3 07/30 Write block 369 len 28672 data: 3 3 3 4 4 2 2 08/30 Flush 09/30 Write block 57 len 24576 data: 4 5 5 5 5 6 10/30 Read block 22 len 8192 11/30 Flush 12/30 Write block 411 len 16384 data: 2 3 3 5 13/30 Write block 408 len 32768 data: 2 2 2 3 4 4 6 5 14/30 Write block 106 len 16384 data: 2 2 2 2 15/30 Write block 314 len 24576 data: 3 3 3 3 3 2 16/30 Read block 212 len 12288 17/30 Read block 3 len 4096 18/30 Write block 496 len 20480 data: 2 2 2 2 2 19/30 Write block 244 len 28672 data: 4 4 3 4 4 4 4 20/30 Write block 28 len 8192 data: 2 2 21/30 Write block 578 len 12288 data: 3 4 4 22/30 Read block 547 len 24576 23/30 Read block 432 len 32768 24/30 Write block 185 len 16384 data: 3 3 3 3 25/30 Write block 174 len 12288 data: 2 3 3 26/30 Read block 565 len 4096 27/30 Read block 18 len 16384 28/30 Write block 105 len 20480 data: 2 3 3 3 3 29/30 Read block 167 len 40960 30/30 Read block 227 len 36864 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 4 Done Done Done false 1012 Acked Write 8 Done Done Done false 1013 Acked Write 4 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 19 18 18 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:31:14.97676239Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1243} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 20 20 20 F F F 1 020-039 12 12 12 20 20 20 T T T 2 040-059 14 14 14 23 23 23 F F F 3 060-079 14 14 14 23 23 23 F F F 4 080-099 12 12 12 20 20 20 F F F 5 100-119 11 11 11 15 15 15 T T T 6 120-139 12 12 12 17 17 17 F F F 7 140-159 10 10 10 14 14 14 F F F 8 160-179 10 10 10 14 14 14 T T T 9 180-199 8 8 8 10 10 10 T T T 10 200-219 12 12 12 18 18 18 F F F 11 220-239 10 10 10 12 12 12 F F F 12 240-259 14 14 14 22 22 22 T T T 13 260-279 10 10 10 12 12 12 F F F 14 280-299 6 6 6 7 7 7 F F F 15 300-319 12 12 12 20 20 20 T T T 16 320-339 14 14 14 22 22 22 F F F 17 340-359 14 14 14 22 22 22 F F F 18 360-379 14 14 14 22 22 22 F F F 19 380-399 10 10 10 12 12 12 F F F 20 400-419 9 9 9 11 11 11 T T T 21 420-439 13 13 13 21 21 21 F F F 22 440-459 12 12 12 19 19 19 F F F 23 460-479 12 12 12 19 19 19 F F F 24 480-499 1 1 1 1 1 1 T T T 25 500-519 11 11 11 15 15 15 T T T 26 520-539 10 10 10 14 14 14 F F F 27 540-559 12 12 12 20 20 20 F F F 28 560-579 14 14 14 22 22 22 T T T 29 580-599 14 14 14 22 22 22 T T T Max gen: 14, Max flush: 23 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 20 20 20 F F F 1 020-039 15 15 15 24 24 24 F F F 2 040-059 14 14 14 23 23 23 F F F 3 060-079 14 14 14 23 23 23 F F F 4 080-099 12 12 12 20 20 20 F F F 5 100-119 15 15 15 24 24 24 F F F 6 120-139 12 12 12 17 17 17 F F F 7 140-159 10 10 10 14 14 14 F F F 8 160-179 15 15 15 24 24 24 F F F 9 180-199 15 15 15 24 24 24 F F F 10 200-219 12 12 12 18 18 18 F F F 11 220-239 10 10 10 12 12 12 F F F 12 240-259 15 15 15 24 24 24 F F F 13 260-279 10 10 10 12 12 12 F F F 14 280-299 6 6 6 7 7 7 F F F 15 300-319 15 15 15 24 24 24 F F F 16 320-339 14 14 14 22 22 22 F F F 17 340-359 14 14 14 22 22 22 F F F 18 360-379 14 14 14 22 22 22 F F F 19 380-399 10 10 10 12 12 12 F F F 20 400-419 15 15 15 24 24 24 F F F 21 420-439 13 13 13 21 21 21 F F F 22 440-459 12 12 12 19 19 19 F F F 23 460-479 12 12 12 19 19 19 F F F 24 480-499 15 15 15 24 24 24 F F F 25 500-519 15 15 15 24 24 24 F F F 26 520-539 10 10 10 14 14 14 F F F 27 540-559 12 12 12 20 20 20 F F F 28 560-579 15 15 15 24 24 24 F F F 29 580-599 15 15 15 24 24 24 F F F Max gen: 15, Max flush: 24 Begin loop 8 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.271269232Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:31:19.312847961Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.313216947Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.344351841Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.345014114Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.345060867Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.345084623Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.345103461Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251} {"msg":"Crucible 903a571b-5de8-4acc-a55d-aee7a393ef87 has session id: ef0f6403-3bd6-4e87-915a-b5dd14d56fe0","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.345498663Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"Upstairs opts: Upstairs UUID: 903a571b-5de8-4acc-a55d-aee7a393ef87, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.345536857Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"Crucible stats registered with UUID: 903a571b-5de8-4acc-a55d-aee7a393ef87","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.345553345Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.376216898Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.376244364Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251} {"msg":"903a571b-5de8-4acc-a55d-aee7a393ef87 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.376308674Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.376327801Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"client":"0","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.376352758Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"client":"0","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.376379484Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"client":"1","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.37640819Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"client":"1","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.376438985Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"client":"2","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.376466551Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"client":"2","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.376495967Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"":"io task","client":"2","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.376522393Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"":"io task","client":"0","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.376537751Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"":"io task","client":"1","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"ds_connection connected from Ok(127.0.0.1:33477)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.3767513Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"":"io task","client":"2","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"ds_connection connected from Ok(127.0.0.1:44304)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.376791674Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"":"io task","client":"1","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"ds_connection connected from Ok(127.0.0.1:44434)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.376821819Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"":"io task","client":"0","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.377470345Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"client":"2","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.37749929Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"client":"2","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.377514188Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"client":"1","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.377528086Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"client":"1","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.377543204Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"client":"0","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.37756837Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"client":"0","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.378164583Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"client":"1","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.378193579Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"client":"2","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.378211426Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"client":"0","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.378768875Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"client":"1","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.378812399Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"client":"1","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.378852283Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.378875329Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"client":"2","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.378900026Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"client":"2","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.378923753Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.378946929Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"client":"0","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.378970126Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"client":"0","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"[0]R flush_numbers[0..12]: [20, 24, 23, 23, 20, 24, 17, 14, 24, 24, 18, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.378993322Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"":"downstairs","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"[0]R generation[0..12]: [12, 15, 14, 14, 12, 15, 12, 10, 15, 15, 12, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.379019348Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"":"downstairs","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.379047255Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"":"downstairs","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"[1]R flush_numbers[0..12]: [20, 24, 23, 23, 20, 24, 17, 14, 24, 24, 18, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.37907548Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"":"downstairs","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"[1]R generation[0..12]: [12, 15, 14, 14, 12, 15, 12, 10, 15, 15, 12, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.379101836Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"":"downstairs","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.379130342Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"":"downstairs","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"[2]R flush_numbers[0..12]: [20, 24, 23, 23, 20, 24, 17, 14, 24, 24, 18, 12]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:31:19.379159528Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"":"downstairs","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"[2]R generation[0..12]: [12, 15, 14, 14, 12, 15, 12, 10, 15, 15, 12, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.379195663Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"":"downstairs","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.379227778Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"":"downstairs","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.379257294Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"":"downstairs","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.379286449Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"":"downstairs","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"Next flush: 25","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.379316735Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"":"downstairs","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.379345911Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ","pid":1251,"":"downstairs","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.379377106Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.379390804Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"client":"0","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.379404682Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"client":"1","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.3794184Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"client":"2","":"downstairs","session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.379432068Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.379444577Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"903a571b-5de8-4acc-a55d-aee7a393ef87 is now active with session: ef0f6403-3bd6-4e87-915a-b5dd14d56fe0","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.379458534Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.379471093Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251,"session_id":"ef0f6403-3bd6-4e87-915a-b5dd14d56fe0"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:19.379483431Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 417 len 8192 data: 5 5 02/30 Write block 450 len 4096 data: 4 03/30 Read block 135 len 8192 04/30 Read block 376 len 40960 05/30 Read block 171 len 32768 06/30 Write block 280 len 40960 data: 2 2 3 3 3 3 2 2 2 2 07/30 Read block 306 len 12288 08/30 Read block 297 len 16384 09/30 Read block 506 len 16384 10/30 Write block 81 len 4096 data: 2 11/30 Flush 12/30 Write block 193 len 20480 data: 2 2 2 2 2 13/30 Flush 14/30 Write block 300 len 12288 data: 3 3 3 15/30 Write block 252 len 4096 data: 5 16/30 Write block 184 len 24576 data: 3 4 4 4 4 3 17/30 Read block 565 len 16384 18/30 Write block 338 len 36864 data: 2 3 3 3 3 2 2 2 2 19/30 Write block 467 len 4096 data: 2 20/30 Write block 193 len 40960 data: 3 3 3 3 3 2 3 3 3 3 21/30 Write block 17 len 32768 data: 2 2 3 3 3 3 3 3 22/30 Write block 573 len 4096 data: 3 23/30 Write block 64 len 20480 data: 5 3 3 4 3 24/30 Write block 143 len 8192 data: 3 3 25/30 Write block 526 len 36864 data: 7 7 6 5 3 3 3 2 2 26/30 Write block 139 len 36864 data: 6 5 4 4 4 4 3 3 3 27/30 Read block 400 len 36864 28/30 Write block 75 len 36864 data: 2 2 2 2 2 2 3 2 3 29/30 Read block 415 len 28672 30/30 Write block 118 len 40960 data: 5 6 6 6 6 6 5 4 3 3 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 3 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:31:19.397080681Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1251} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 20 20 20 T T T 1 020-039 15 15 15 24 24 24 T T T 2 040-059 14 14 14 23 23 23 F F F 3 060-079 14 14 14 23 23 23 T T T 4 080-099 16 16 16 25 25 25 T T T 5 100-119 15 15 15 24 24 24 T T T 6 120-139 12 12 12 17 17 17 T T T 7 140-159 10 10 10 14 14 14 T T T 8 160-179 15 15 15 24 24 24 F F F 9 180-199 16 16 16 26 26 26 T T T 10 200-219 12 12 12 18 18 18 T T T 11 220-239 10 10 10 12 12 12 F F F 12 240-259 15 15 15 24 24 24 T T T 13 260-279 10 10 10 12 12 12 F F F 14 280-299 16 16 16 25 25 25 F F F 15 300-319 15 15 15 24 24 24 T T T 16 320-339 14 14 14 22 22 22 T T T 17 340-359 14 14 14 22 22 22 T T T 18 360-379 14 14 14 22 22 22 F F F 19 380-399 10 10 10 12 12 12 F F F 20 400-419 16 16 16 25 25 25 F F F 21 420-439 13 13 13 21 21 21 F F F 22 440-459 16 16 16 25 25 25 F F F 23 460-479 12 12 12 19 19 19 T T T 24 480-499 15 15 15 24 24 24 F F F 25 500-519 15 15 15 24 24 24 F F F 26 520-539 10 10 10 14 14 14 T T T 27 540-559 12 12 12 20 20 20 F F F 28 560-579 15 15 15 24 24 24 T T T 29 580-599 15 15 15 24 24 24 F F F Max gen: 16, Max flush: 26 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 27 27 27 F F F 1 020-039 17 17 17 27 27 27 F F F 2 040-059 14 14 14 23 23 23 F F F 3 060-079 17 17 17 27 27 27 F F F 4 080-099 17 17 17 27 27 27 F F F 5 100-119 17 17 17 27 27 27 F F F 6 120-139 17 17 17 27 27 27 F F F 7 140-159 17 17 17 27 27 27 F F F 8 160-179 15 15 15 24 24 24 F F F 9 180-199 17 17 17 27 27 27 F F F 10 200-219 17 17 17 27 27 27 F F F 11 220-239 10 10 10 12 12 12 F F F 12 240-259 17 17 17 27 27 27 F F F 13 260-279 10 10 10 12 12 12 F F F 14 280-299 16 16 16 25 25 25 F F F 15 300-319 17 17 17 27 27 27 F F F 16 320-339 17 17 17 27 27 27 F F F 17 340-359 17 17 17 27 27 27 F F F 18 360-379 14 14 14 22 22 22 F F F 19 380-399 10 10 10 12 12 12 F F F 20 400-419 16 16 16 25 25 25 F F F 21 420-439 13 13 13 21 21 21 F F F 22 440-459 16 16 16 25 25 25 F F F 23 460-479 17 17 17 27 27 27 F F F 24 480-499 15 15 15 24 24 24 F F F 25 500-519 15 15 15 24 24 24 F F F 26 520-539 17 17 17 27 27 27 F F F 27 540-559 12 12 12 20 20 20 F F F 28 560-579 17 17 17 27 27 27 F F F 29 580-599 15 15 15 24 24 24 F F F Max gen: 17, Max flush: 27 Begin loop 9 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.746038792Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.790076191Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.790721607Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.790756762Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.79077122Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.790785937Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259} {"msg":"Crucible dad344c0-3769-417c-b386-a97b6c6774dd has session id: 23945a28-ee29-4697-9d4e-7b0eb2404c14","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.791237471Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"Upstairs opts: Upstairs UUID: dad344c0-3769-417c-b386-a97b6c6774dd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.791278915Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"Crucible stats registered with UUID: dad344c0-3769-417c-b386-a97b6c6774dd","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.791296823Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.822288518Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.822332951Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259} {"msg":"dad344c0-3769-417c-b386-a97b6c6774dd active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.822370395Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.822401751Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"client":"0","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.822448494Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"client":"0","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.822487829Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"client":"1","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.822524303Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"client":"1","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.822560688Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"client":"2","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.822592633Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"client":"2","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.822609631Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"":"io task","client":"2","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.822624218Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"":"io task","client":"0","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.822648575Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"":"io task","client":"1","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"ds_connection connected from Ok(127.0.0.1:48046)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.82268191Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"":"io task","client":"0","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"ds_connection connected from Ok(127.0.0.1:43374)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.822709666Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"":"io task","client":"1","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"ds_connection connected from Ok(127.0.0.1:50029)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.822874292Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"":"io task","client":"2","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.823424482Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"client":"1","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.823458437Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"client":"1","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.823606765Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"client":"2","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.82364101Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"client":"2","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.823669096Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"client":"0","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.823697622Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"client":"0","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.824107132Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"client":"1","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.824137957Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"client":"0","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.824401599Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"client":"2","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.824714123Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"client":"1","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.824741569Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"client":"1","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.824756527Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.824772585Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"client":"0","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.824786853Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"client":"0","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.824800741Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.824869261Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"client":"2","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.824888628Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"client":"2","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"[0]R flush_numbers[0..12]: [27, 27, 23, 27, 27, 27, 27, 27, 24, 27, 27, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.824902816Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"":"downstairs","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"[0]R generation[0..12]: [17, 17, 14, 17, 17, 17, 17, 17, 15, 17, 17, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.824917134Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"":"downstairs","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.824931572Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"":"downstairs","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"[1]R flush_numbers[0..12]: [27, 27, 23, 27, 27, 27, 27, 27, 24, 27, 27, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.824945579Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"":"downstairs","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"[1]R generation[0..12]: [17, 17, 14, 17, 17, 17, 17, 17, 15, 17, 17, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.824959488Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"":"downstairs","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.824973406Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"":"downstairs","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"[2]R flush_numbers[0..12]: [27, 27, 23, 27, 27, 27, 27, 27, 24, 27, 27, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.824987373Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"":"downstairs","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"[2]R generation[0..12]: [17, 17, 14, 17, 17, 17, 17, 17, 15, 17, 17, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.825001062Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"":"downstairs","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.825015229Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"":"downstairs","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.825031177Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"":"downstairs","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.825044845Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"":"downstairs","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"Next flush: 28","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.825057933Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"":"downstairs","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:31:23.825071041Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"":"downstairs","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.825091858Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.825104067Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"client":"0","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.825117654Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"client":"1","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.825131052Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"client":"2","":"downstairs","session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.82514463Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.825157109Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"dad344c0-3769-417c-b386-a97b6c6774dd is now active with session: 23945a28-ee29-4697-9d4e-7b0eb2404c14","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.825170067Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.825182275Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259,"session_id":"23945a28-ee29-4697-9d4e-7b0eb2404c14"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:23.825194613Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 404 len 20480 data: 3 3 3 2 3 02/30 Write block 229 len 24576 data: 2 2 2 2 2 2 03/30 Read block 250 len 36864 04/30 Read block 397 len 28672 05/30 Read block 369 len 12288 06/30 Read block 17 len 28672 07/30 Write block 494 len 8192 data: 2 2 08/30 Write block 344 len 36864 data: 3 3 3 2 2 2 2 3 3 09/30 Read block 309 len 16384 10/30 Write block 207 len 4096 data: 3 11/30 Flush 12/30 Write block 255 len 20480 data: 4 4 4 3 4 13/30 Write block 435 len 32768 data: 2 2 2 2 2 2 2 2 14/30 Write block 39 len 4096 data: 2 15/30 Write block 147 len 24576 data: 4 2 2 2 2 2 16/30 Read block 537 len 40960 17/30 Read block 3 len 28672 18/30 Write block 56 len 12288 data: 3 5 6 19/30 Write block 365 len 16384 data: 4 4 3 3 20/30 Write block 64 len 32768 data: 6 4 4 5 4 2 2 2 21/30 Read block 0 len 20480 22/30 Read block 509 len 20480 23/30 Read block 458 len 12288 24/30 Read block 221 len 16384 25/30 Read block 474 len 32768 26/30 Write block 411 len 24576 data: 4 5 5 7 6 5 27/30 Read block 261 len 28672 28/30 Read block 68 len 24576 29/30 Write block 268 len 16384 data: 3 2 2 2 30/30 Write block 396 len 28672 data: 3 3 3 3 3 3 3 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 5 Done Done Done false 1012 Acked Write 8 Done Done Done false 1013 Acked Write 1 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:31:23.843483083Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1259} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 27 27 27 F F F 1 020-039 17 17 17 27 27 27 T T T 2 040-059 14 14 14 23 23 23 T T T 3 060-079 17 17 17 27 27 27 T T T 4 080-099 17 17 17 27 27 27 F F F 5 100-119 17 17 17 27 27 27 F F F 6 120-139 17 17 17 27 27 27 F F F 7 140-159 17 17 17 27 27 27 T T T 8 160-179 15 15 15 24 24 24 F F F 9 180-199 17 17 17 27 27 27 F F F 10 200-219 18 18 18 28 28 28 F F F 11 220-239 18 18 18 28 28 28 F F F 12 240-259 17 17 17 27 27 27 T T T 13 260-279 10 10 10 12 12 12 T T T 14 280-299 16 16 16 25 25 25 F F F 15 300-319 17 17 17 27 27 27 F F F 16 320-339 17 17 17 27 27 27 F F F 17 340-359 18 18 18 28 28 28 F F F 18 360-379 14 14 14 22 22 22 T T T 19 380-399 10 10 10 12 12 12 T T T 20 400-419 18 18 18 28 28 28 T T T 21 420-439 13 13 13 21 21 21 T T T 22 440-459 16 16 16 25 25 25 T T T 23 460-479 17 17 17 27 27 27 F F F 24 480-499 18 18 18 28 28 28 F F F 25 500-519 15 15 15 24 24 24 F F F 26 520-539 17 17 17 27 27 27 F F F 27 540-559 12 12 12 20 20 20 F F F 28 560-579 17 17 17 27 27 27 F F F 29 580-599 15 15 15 24 24 24 F F F Max gen: 18, Max flush: 28 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 27 27 27 F F F 1 020-039 19 19 19 29 29 29 F F F 2 040-059 19 19 19 29 29 29 F F F 3 060-079 19 19 19 29 29 29 F F F 4 080-099 17 17 17 27 27 27 F F F 5 100-119 17 17 17 27 27 27 F F F 6 120-139 17 17 17 27 27 27 F F F 7 140-159 19 19 19 29 29 29 F F F 8 160-179 15 15 15 24 24 24 F F F 9 180-199 17 17 17 27 27 27 F F F 10 200-219 18 18 18 28 28 28 F F F 11 220-239 18 18 18 28 28 28 F F F 12 240-259 19 19 19 29 29 29 F F F 13 260-279 19 19 19 29 29 29 F F F 14 280-299 16 16 16 25 25 25 F F F 15 300-319 17 17 17 27 27 27 F F F 16 320-339 17 17 17 27 27 27 F F F 17 340-359 18 18 18 28 28 28 F F F 18 360-379 19 19 19 29 29 29 F F F 19 380-399 19 19 19 29 29 29 F F F 20 400-419 19 19 19 29 29 29 F F F 21 420-439 19 19 19 29 29 29 F F F 22 440-459 19 19 19 29 29 29 F F F 23 460-479 17 17 17 27 27 27 F F F 24 480-499 18 18 18 28 28 28 F F F 25 500-519 15 15 15 24 24 24 F F F 26 520-539 17 17 17 27 27 27 F F F 27 540-559 12 12 12 20 20 20 F F F 28 560-579 17 17 17 27 27 27 F F F 29 580-599 15 15 15 24 24 24 F F F Max gen: 19, Max flush: 29 Begin loop 10 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.140481051Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.184145035Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.184750217Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.184787472Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.184802519Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.184817417Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267} {"msg":"Crucible 5d3d1e61-acd2-4e8b-981c-664830be52f3 has session id: c51ca6a2-a56e-4aef-8729-139e234fa012","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.185229397Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"Upstairs opts: Upstairs UUID: 5d3d1e61-acd2-4e8b-981c-664830be52f3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.185276221Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"Crucible stats registered with UUID: 5d3d1e61-acd2-4e8b-981c-664830be52f3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.185301147Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.215717605Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.21575539Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267} {"msg":"5d3d1e61-acd2-4e8b-981c-664830be52f3 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.215789745Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.215807563Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"client":"0","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.215835019Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"client":"0","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.215852796Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"client":"1","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.215869903Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"client":"1","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.215886601Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"client":"2","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.215903518Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"client":"2","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.215920136Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"":"io task","client":"0","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.215937533Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"":"io task","client":"2","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.215954611Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"":"io task","client":"1","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"ds_connection connected from Ok(127.0.0.1:42669)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.216040009Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"":"io task","client":"2","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"ds_connection connected from Ok(127.0.0.1:48691)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.216079733Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"":"io task","client":"1","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"ds_connection connected from Ok(127.0.0.1:42460)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.216277274Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"":"io task","client":"0","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.216928969Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"client":"2","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.216963634Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"client":"2","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.216983891Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"client":"0","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.217000508Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"client":"0","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.217067128Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"client":"1","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.217111882Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"client":"1","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.217501555Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"client":"2","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.217568885Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"client":"1","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.217786663Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"client":"0","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.21808163Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"client":"2","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.218107407Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"client":"2","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.218122365Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.21821959Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"client":"1","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.218256495Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"client":"1","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.218281741Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.218377447Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"client":"0","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.218396454Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"client":"0","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"[0]R flush_numbers[0..12]: [27, 29, 29, 29, 27, 27, 27, 29, 24, 27, 28, 28]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.218410302Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"":"downstairs","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"[0]R generation[0..12]: [17, 19, 19, 19, 17, 17, 17, 19, 15, 17, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.21842429Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"":"downstairs","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.218438798Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"":"downstairs","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"[1]R flush_numbers[0..12]: [27, 29, 29, 29, 27, 27, 27, 29, 24, 27, 28, 28]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.218452516Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"":"downstairs","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"[1]R generation[0..12]: [17, 19, 19, 19, 17, 17, 17, 19, 15, 17, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.218466194Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"":"downstairs","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.218480592Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"":"downstairs","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"[2]R flush_numbers[0..12]: [27, 29, 29, 29, 27, 27, 27, 29, 24, 27, 28, 28]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.21849448Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"":"downstairs","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"[2]R generation[0..12]: [17, 19, 19, 19, 17, 17, 17, 19, 15, 17, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.218508428Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"":"downstairs","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.218522286Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"":"downstairs","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.218535534Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"":"downstairs","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.218548972Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"":"downstairs","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"Next flush: 30","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.21856493Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"":"downstairs","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:31:28.218578268Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"":"downstairs","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.218597665Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.218609853Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"client":"0","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.218623271Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"client":"1","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.218636529Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"client":"2","":"downstairs","session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.218649907Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.218662346Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"5d3d1e61-acd2-4e8b-981c-664830be52f3 is now active with session: c51ca6a2-a56e-4aef-8729-139e234fa012","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.218675274Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.218687542Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267,"session_id":"c51ca6a2-a56e-4aef-8729-139e234fa012"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:28.21870019Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 228 len 36864 data: 2 3 3 3 3 3 3 2 2 02/30 Write block 21 len 16384 data: 4 4 4 4 03/30 Read block 11 len 24576 04/30 Read block 537 len 20480 05/30 Write block 70 len 24576 data: 3 3 2 2 2 3 06/30 Write block 495 len 40960 data: 3 3 3 3 3 3 2 3 4 4 07/30 Write block 182 len 40960 data: 4 3 4 5 5 5 5 4 4 3 08/30 Read block 157 len 40960 09/30 Write block 556 len 40960 data: 3 3 3 2 2 2 3 3 3 3 10/30 Write block 498 len 20480 data: 4 4 4 3 4 11/30 Write block 57 len 12288 data: 6 7 6 12/30 Read block 463 len 12288 13/30 Read block 226 len 16384 14/30 Read block 497 len 20480 15/30 Write block 529 len 40960 data: 6 4 4 4 3 3 2 2 2 2 16/30 Read block 203 len 32768 17/30 Write block 130 len 32768 data: 4 3 3 2 3 4 4 4 18/30 Read block 535 len 24576 19/30 Write block 392 len 40960 data: 4 4 4 3 4 4 4 4 4 4 20/30 Read block 569 len 20480 21/30 Flush 22/30 Write block 221 len 16384 data: 3 3 4 3 23/30 Write block 358 len 20480 data: 5 5 5 4 4 24/30 Write block 233 len 16384 data: 4 4 3 3 25/30 Read block 115 len 28672 26/30 Read block 370 len 24576 27/30 Write block 89 len 8192 data: 3 3 28/30 Read block 92 len 24576 29/30 Read block 413 len 12288 30/30 Write block 14 len 24576 data: 3 2 2 3 3 4 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 4 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:31:28.239765484Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1267} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 27 27 27 T T T 1 020-039 20 20 20 30 30 30 F F F 2 040-059 20 20 20 30 30 30 F F F 3 060-079 20 20 20 30 30 30 F F F 4 080-099 17 17 17 27 27 27 T T T 5 100-119 17 17 17 27 27 27 F F F 6 120-139 20 20 20 30 30 30 F F F 7 140-159 19 19 19 29 29 29 F F F 8 160-179 15 15 15 24 24 24 F F F 9 180-199 20 20 20 30 30 30 F F F 10 200-219 18 18 18 28 28 28 F F F 11 220-239 20 20 20 30 30 30 T T T 12 240-259 19 19 19 29 29 29 F F F 13 260-279 19 19 19 29 29 29 F F F 14 280-299 16 16 16 25 25 25 F F F 15 300-319 17 17 17 27 27 27 F F F 16 320-339 17 17 17 27 27 27 F F F 17 340-359 18 18 18 28 28 28 T T T 18 360-379 19 19 19 29 29 29 T T T 19 380-399 20 20 20 30 30 30 F F F 20 400-419 20 20 20 30 30 30 F F F 21 420-439 19 19 19 29 29 29 F F F 22 440-459 19 19 19 29 29 29 F F F 23 460-479 17 17 17 27 27 27 F F F 24 480-499 20 20 20 30 30 30 F F F 25 500-519 20 20 20 30 30 30 F F F 26 520-539 20 20 20 30 30 30 F F F 27 540-559 20 20 20 30 30 30 F F F 28 560-579 20 20 20 30 30 30 F F F 29 580-599 15 15 15 24 24 24 F F F Max gen: 20, Max flush: 30 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 31 31 31 F F F 1 020-039 20 20 20 30 30 30 F F F 2 040-059 20 20 20 30 30 30 F F F 3 060-079 20 20 20 30 30 30 F F F 4 080-099 21 21 21 31 31 31 F F F 5 100-119 17 17 17 27 27 27 F F F 6 120-139 20 20 20 30 30 30 F F F 7 140-159 19 19 19 29 29 29 F F F 8 160-179 15 15 15 24 24 24 F F F 9 180-199 20 20 20 30 30 30 F F F 10 200-219 18 18 18 28 28 28 F F F 11 220-239 21 21 21 31 31 31 F F F 12 240-259 19 19 19 29 29 29 F F F 13 260-279 19 19 19 29 29 29 F F F 14 280-299 16 16 16 25 25 25 F F F 15 300-319 17 17 17 27 27 27 F F F 16 320-339 17 17 17 27 27 27 F F F 17 340-359 21 21 21 31 31 31 F F F 18 360-379 21 21 21 31 31 31 F F F 19 380-399 20 20 20 30 30 30 F F F 20 400-419 20 20 20 30 30 30 F F F 21 420-439 19 19 19 29 29 29 F F F 22 440-459 19 19 19 29 29 29 F F F 23 460-479 17 17 17 27 27 27 F F F 24 480-499 20 20 20 30 30 30 F F F 25 500-519 20 20 20 30 30 30 F F F 26 520-539 20 20 20 30 30 30 F F F 27 540-559 20 20 20 30 30 30 F F F 28 560-579 20 20 20 30 30 30 F F F 29 580-599 15 15 15 24 24 24 F F F Max gen: 21, Max flush: 31 Begin loop 11 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.508987858Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.553465784Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.554090673Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.554127147Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.554142065Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.554157233Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275} {"msg":"Crucible 30aa366d-aa4c-4cee-9c34-5434f50481a6 has session id: 7b54fe0a-3e99-4351-88bf-7f860d71b722","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.554607397Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"Upstairs opts: Upstairs UUID: 30aa366d-aa4c-4cee-9c34-5434f50481a6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.554651871Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"Crucible stats registered with UUID: 30aa366d-aa4c-4cee-9c34-5434f50481a6","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.554677867Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.585221127Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.585259922Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275} {"msg":"30aa366d-aa4c-4cee-9c34-5434f50481a6 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.585324982Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.585343779Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"client":"0","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.585366366Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"client":"0","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.585380544Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"client":"1","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.585394382Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"client":"1","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.58540785Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"client":"2","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.585421608Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"client":"2","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.585434976Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"":"io task","client":"2","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.585448854Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"":"io task","client":"0","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.585462672Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"":"io task","client":"1","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"ds_connection connected from Ok(127.0.0.1:61100)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.585572046Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"":"io task","client":"2","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"ds_connection connected from Ok(127.0.0.1:65400)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.585627858Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"":"io task","client":"1","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"ds_connection connected from Ok(127.0.0.1:62984)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.585775226Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"":"io task","client":"0","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.586443089Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"client":"2","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.586480213Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"client":"2","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.586508779Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"client":"0","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.586536785Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"client":"0","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.586643799Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"client":"1","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.586687733Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"client":"1","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.587200178Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"client":"0","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.587279236Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"client":"2","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.58732503Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"client":"1","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.587785213Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"client":"0","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.587816658Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"client":"0","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.587831456Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.587844484Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"client":"2","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.587858302Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"client":"2","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.58787187Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.587884298Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"client":"1","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.587897656Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"client":"1","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"[0]R flush_numbers[0..12]: [31, 30, 30, 30, 31, 27, 30, 29, 24, 30, 28, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.587915074Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"":"downstairs","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"[0]R generation[0..12]: [21, 20, 20, 20, 21, 17, 20, 19, 15, 20, 18, 21]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.587929372Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"":"downstairs","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.58794363Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"":"downstairs","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"[1]R flush_numbers[0..12]: [31, 30, 30, 30, 31, 27, 30, 29, 24, 30, 28, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.587957417Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"":"downstairs","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"[1]R generation[0..12]: [21, 20, 20, 20, 21, 17, 20, 19, 15, 20, 18, 21]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.587971115Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"":"downstairs","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.587985293Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"":"downstairs","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"[2]R flush_numbers[0..12]: [31, 30, 30, 30, 31, 27, 30, 29, 24, 30, 28, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.588015359Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"":"downstairs","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"[2]R generation[0..12]: [21, 20, 20, 20, 21, 17, 20, 19, 15, 20, 18, 21]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.588030427Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"":"downstairs","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.588044585Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"":"downstairs","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.588057753Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"":"downstairs","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:31:32.588070891Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"":"downstairs","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"Next flush: 32","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.588094757Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"":"downstairs","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.588107766Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"":"downstairs","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.588121114Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.588135602Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"client":"0","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.588149509Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"client":"1","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.588163107Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"client":"2","":"downstairs","session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.588176616Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.588188954Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"30aa366d-aa4c-4cee-9c34-5434f50481a6 is now active with session: 7b54fe0a-3e99-4351-88bf-7f860d71b722","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.588201872Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.58821409Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275,"session_id":"7b54fe0a-3e99-4351-88bf-7f860d71b722"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:32.588226748Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 372 len 16384 data: 5 5 3 3 02/30 Write block 427 len 24576 data: 3 3 3 2 2 2 03/30 Read block 177 len 4096 04/30 Write block 477 len 36864 data: 2 2 2 2 2 2 2 2 2 05/30 Write block 458 len 32768 data: 3 3 3 3 3 3 3 3 06/30 Flush 07/30 Write block 584 len 20480 data: 4 3 3 3 3 08/30 Read block 330 len 36864 09/30 Read block 154 len 12288 10/30 Write block 75 len 8192 data: 4 3 11/30 Write block 106 len 16384 data: 4 4 4 4 12/30 Write block 344 len 16384 data: 4 4 4 3 13/30 Write block 176 len 36864 data: 4 3 3 3 3 3 5 4 5 14/30 Read block 460 len 36864 15/30 Flush 16/30 Write block 435 len 20480 data: 3 3 3 3 3 17/30 Read block 316 len 20480 18/30 Write block 103 len 24576 data: 4 3 3 5 5 5 19/30 Write block 524 len 20480 data: 5 7 8 8 7 20/30 Flush 21/30 Write block 99 len 36864 data: 5 3 3 4 5 4 4 6 6 22/30 Write block 99 len 8192 data: 6 4 23/30 Read block 313 len 36864 24/30 Write block 521 len 12288 data: 4 5 5 25/30 Write block 469 len 32768 data: 2 2 2 2 2 2 2 2 26/30 Write block 292 len 16384 data: 2 3 3 2 27/30 Read block 417 len 24576 28/30 Read block 308 len 32768 29/30 Read block 473 len 20480 30/30 Read block 352 len 24576 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 9 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 6 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 10 9 10 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:31:32.607421006Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1275} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 31 31 31 F F F 1 020-039 20 20 20 30 30 30 F F F 2 040-059 20 20 20 30 30 30 F F F 3 060-079 22 22 22 33 33 33 F F F 4 080-099 21 21 21 31 31 31 T T T 5 100-119 22 22 22 34 34 34 T T T 6 120-139 20 20 20 30 30 30 F F F 7 140-159 19 19 19 29 29 29 F F F 8 160-179 22 22 22 33 33 33 F F F 9 180-199 22 22 22 33 33 33 F F F 10 200-219 18 18 18 28 28 28 F F F 11 220-239 21 21 21 31 31 31 F F F 12 240-259 19 19 19 29 29 29 F F F 13 260-279 19 19 19 29 29 29 F F F 14 280-299 16 16 16 25 25 25 T T T 15 300-319 17 17 17 27 27 27 F F F 16 320-339 17 17 17 27 27 27 F F F 17 340-359 22 22 22 33 33 33 F F F 18 360-379 22 22 22 32 32 32 F F F 19 380-399 20 20 20 30 30 30 F F F 20 400-419 20 20 20 30 30 30 F F F 21 420-439 22 22 22 34 34 34 F F F 22 440-459 22 22 22 32 32 32 F F F 23 460-479 22 22 22 32 32 32 T T T 24 480-499 22 22 22 32 32 32 F F F 25 500-519 20 20 20 30 30 30 F F F 26 520-539 22 22 22 34 34 34 T T T 27 540-559 20 20 20 30 30 30 F F F 28 560-579 20 20 20 30 30 30 F F F 29 580-599 22 22 22 33 33 33 F F F Max gen: 22, Max flush: 34 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 31 31 31 F F F 1 020-039 20 20 20 30 30 30 F F F 2 040-059 20 20 20 30 30 30 F F F 3 060-079 22 22 22 33 33 33 F F F 4 080-099 23 23 23 35 35 35 F F F 5 100-119 23 23 23 35 35 35 F F F 6 120-139 20 20 20 30 30 30 F F F 7 140-159 19 19 19 29 29 29 F F F 8 160-179 22 22 22 33 33 33 F F F 9 180-199 22 22 22 33 33 33 F F F 10 200-219 18 18 18 28 28 28 F F F 11 220-239 21 21 21 31 31 31 F F F 12 240-259 19 19 19 29 29 29 F F F 13 260-279 19 19 19 29 29 29 F F F 14 280-299 23 23 23 35 35 35 F F F 15 300-319 17 17 17 27 27 27 F F F 16 320-339 17 17 17 27 27 27 F F F 17 340-359 22 22 22 33 33 33 F F F 18 360-379 22 22 22 32 32 32 F F F 19 380-399 20 20 20 30 30 30 F F F 20 400-419 20 20 20 30 30 30 F F F 21 420-439 22 22 22 34 34 34 F F F 22 440-459 22 22 22 32 32 32 F F F 23 460-479 23 23 23 35 35 35 F F F 24 480-499 22 22 22 32 32 32 F F F 25 500-519 20 20 20 30 30 30 F F F 26 520-539 23 23 23 35 35 35 F F F 27 540-559 20 20 20 30 30 30 F F F 28 560-579 20 20 20 30 30 30 F F F 29 580-599 22 22 22 33 33 33 F F F Max gen: 23, Max flush: 35 Begin loop 12 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.872323059Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.915881969Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.916479462Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.916517296Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.916532334Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.916547401Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283} {"msg":"Crucible 8c13ae2a-aa4f-44cc-b263-08499ddbba24 has session id: a2d4a992-9910-4734-9e7a-0e286e4913ff","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.917021192Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"Upstairs opts: Upstairs UUID: 8c13ae2a-aa4f-44cc-b263-08499ddbba24, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.917093282Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"Crucible stats registered with UUID: 8c13ae2a-aa4f-44cc-b263-08499ddbba24","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.917127067Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.948380983Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.948425347Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283} {"msg":"8c13ae2a-aa4f-44cc-b263-08499ddbba24 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.948464011Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.948496047Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"client":"0","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.948549759Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"client":"0","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.948589703Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"client":"1","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.948621458Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"client":"1","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.948654164Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"client":"2","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.948685819Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"client":"2","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.948717994Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"":"io task","client":"0","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.948749Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"":"io task","client":"2","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.948769956Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"":"io task","client":"1","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:48786)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.948802742Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"":"io task","client":"1","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:51059)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.948984175Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"":"io task","client":"2","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:61000)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.949023659Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"":"io task","client":"0","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.949606254Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"client":"1","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.94963707Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"client":"1","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.949652818Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"client":"0","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.949727067Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"client":"0","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.949745894Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"client":"2","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.949764701Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"client":"2","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.950209886Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"client":"1","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.950247121Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"client":"2","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.950355345Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"client":"0","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.950815738Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"client":"1","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.950844504Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"client":"1","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.950859751Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.950872409Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"client":"2","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.950886238Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"client":"2","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.950900616Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.950913204Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"client":"0","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.950926902Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"client":"0","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"[0]R flush_numbers[0..12]: [31, 30, 30, 33, 35, 35, 30, 29, 33, 33, 28, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.950940799Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"":"downstairs","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"[0]R generation[0..12]: [21, 20, 20, 22, 23, 23, 20, 19, 22, 22, 18, 21]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.950954828Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"":"downstairs","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.950969215Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"":"downstairs","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"[1]R flush_numbers[0..12]: [31, 30, 30, 33, 35, 35, 30, 29, 33, 33, 28, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.950987093Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"":"downstairs","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"[1]R generation[0..12]: [21, 20, 20, 22, 23, 23, 20, 19, 22, 22, 18, 21]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.951001411Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"":"downstairs","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.951015869Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"":"downstairs","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"[2]R flush_numbers[0..12]: [31, 30, 30, 33, 35, 35, 30, 29, 33, 33, 28, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.951029737Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"":"downstairs","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"[2]R generation[0..12]: [21, 20, 20, 22, 23, 23, 20, 19, 22, 22, 18, 21]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.951043275Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"":"downstairs","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {Wait for a query_work_queue command to finish before sending IO "msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.951065272Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"":"downstairs","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.95107881Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"":"downstairs","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.951092317Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"":"downstairs","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"Next flush: 36","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.951105415Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"":"downstairs","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.951118574Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"":"downstairs","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.951131942Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.95114408Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"client":"0","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.951157748Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"client":"1","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.951171156Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"client":"2","":"downstairs","session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.951184664Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.951199222Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"8c13ae2a-aa4f-44cc-b263-08499ddbba24 is now active with session: a2d4a992-9910-4734-9e7a-0e286e4913ff","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.95121246Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.951224798Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283,"session_id":"a2d4a992-9910-4734-9e7a-0e286e4913ff"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:36.951237336Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 206 len 8192 data: 3 4 02/30 Read block 579 len 40960 03/30 Write block 370 len 8192 data: 4 4 04/30 Read block 366 len 20480 05/30 Read block 140 len 12288 06/30 Write block 223 len 24576 data: 5 4 2 2 2 3 07/30 Write block 533 len 4096 data: 4 08/30 Read block 391 len 4096 09/30 Read block 63 len 8192 10/30 Read block 310 len 8192 11/30 Read block 542 len 36864 12/30 Write block 577 len 28672 data: 3 4 5 5 4 4 4 13/30 Write block 588 len 36864 data: 4 3 3 3 3 3 3 3 2 14/30 Write block 363 len 36864 data: 4 4 5 5 4 4 4 5 5 15/30 Read block 547 len 12288 16/30 Read block 37 len 28672 17/30 Read block 132 len 4096 18/30 Read block 238 len 4096 19/30 Flush 20/30 Write block 361 len 20480 data: 5 5 5 5 6 21/30 Flush 22/30 Write block 520 len 32768 data: 2 5 6 6 6 8 9 9 23/30 Write block 123 len 16384 data: 7 6 5 4 24/30 Read block 346 len 36864 25/30 Read block 435 len 12288 26/30 Write block 326 len 36864 data: 2 2 2 2 2 2 2 2 2 27/30 Read block 128 len 32768 28/30 Write block 197 len 32768 data: 4 3 4 4 4 4 3 3 29/30 Read block 553 len 16384 30/30 Read block 338 len 12288 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 8 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 9 9 8 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:31:36.97059555Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1283} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 31 31 31 F F F 1 020-039 20 20 20 30 30 30 F F F 2 040-059 20 20 20 30 30 30 F F F 3 060-079 22 22 22 33 33 33 F F F 4 080-099 23 23 23 35 35 35 F F F 5 100-119 23 23 23 35 35 35 F F F 6 120-139 20 20 20 30 30 30 T T T 7 140-159 19 19 19 29 29 29 F F F 8 160-179 22 22 22 33 33 33 F F F 9 180-199 22 22 22 33 33 33 T T T 10 200-219 24 24 24 36 36 36 T T T 11 220-239 24 24 24 36 36 36 F F F 12 240-259 19 19 19 29 29 29 F F F 13 260-279 19 19 19 29 29 29 F F F 14 280-299 23 23 23 35 35 35 F F F 15 300-319 17 17 17 27 27 27 F F F 16 320-339 17 17 17 27 27 27 T T T 17 340-359 22 22 22 33 33 33 F F F 18 360-379 24 24 24 37 37 37 F F F 19 380-399 20 20 20 30 30 30 F F F 20 400-419 20 20 20 30 30 30 F F F 21 420-439 22 22 22 34 34 34 F F F 22 440-459 22 22 22 32 32 32 F F F 23 460-479 23 23 23 35 35 35 F F F 24 480-499 22 22 22 32 32 32 F F F 25 500-519 20 20 20 30 30 30 F F F 26 520-539 24 24 24 36 36 36 T T T 27 540-559 20 20 20 30 30 30 F F F 28 560-579 24 24 24 36 36 36 F F F 29 580-599 24 24 24 36 36 36 F F F Max gen: 24, Max flush: 37 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 31 31 31 F F F 1 020-039 20 20 20 30 30 30 F F F 2 040-059 20 20 20 30 30 30 F F F 3 060-079 22 22 22 33 33 33 F F F 4 080-099 23 23 23 35 35 35 F F F 5 100-119 23 23 23 35 35 35 F F F 6 120-139 25 25 25 38 38 38 F F F 7 140-159 19 19 19 29 29 29 F F F 8 160-179 22 22 22 33 33 33 F F F 9 180-199 25 25 25 38 38 38 F F F 10 200-219 25 25 25 38 38 38 F F F 11 220-239 24 24 24 36 36 36 F F F 12 240-259 19 19 19 29 29 29 F F F 13 260-279 19 19 19 29 29 29 F F F 14 280-299 23 23 23 35 35 35 F F F 15 300-319 17 17 17 27 27 27 F F F 16 320-339 25 25 25 38 38 38 F F F 17 340-359 22 22 22 33 33 33 F F F 18 360-379 24 24 24 37 37 37 F F F 19 380-399 20 20 20 30 30 30 F F F 20 400-419 20 20 20 30 30 30 F F F 21 420-439 22 22 22 34 34 34 F F F 22 440-459 22 22 22 32 32 32 F F F 23 460-479 23 23 23 35 35 35 F F F 24 480-499 22 22 22 32 32 32 F F F 25 500-519 20 20 20 30 30 30 F F F 26 520-539 25 25 25 38 38 38 F F F 27 540-559 20 20 20 30 30 30 F F F 28 560-579 24 24 24 36 36 36 F F F 29 580-599 24 24 24 36 36 36 F F F Max gen: 25, Max flush: 38 Begin loop 13 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.235847143Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.280338127Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.28100021Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.281049253Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.28107394Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.281092807Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291} {"msg":"Crucible d591e42c-a432-4af8-8584-a1eecb3f91c3 has session id: 13fd48fa-b73c-42ea-b6f2-083416b8fbba","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.281473721Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"Upstairs opts: Upstairs UUID: d591e42c-a432-4af8-8584-a1eecb3f91c3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.281520474Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"Crucible stats registered with UUID: d591e42c-a432-4af8-8584-a1eecb3f91c3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.281540282Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.312184377Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.312216452Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291} {"msg":"d591e42c-a432-4af8-8584-a1eecb3f91c3 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.312250418Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.312267215Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"client":"0","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.312295311Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"client":"0","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.312314558Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"client":"1","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.312332375Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"client":"1","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.312360162Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"client":"2","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.312399986Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"client":"2","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.312425692Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"":"io task","client":"2","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.312454957Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"":"io task","client":"0","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.312487773Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"":"io task","client":"1","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"ds_connection connected from Ok(127.0.0.1:52788)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.31257154Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"":"io task","client":"0","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"ds_connection connected from Ok(127.0.0.1:42303)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.312602796Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"":"io task","client":"2","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"ds_connection connected from Ok(127.0.0.1:51090)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.312793348Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"":"io task","client":"1","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.31339797Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"client":"0","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.313426376Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"client":"0","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.313442033Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"client":"2","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.313460791Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"client":"2","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.313562446Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"client":"1","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.313592571Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"client":"1","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.314098078Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"client":"0","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.314128083Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"client":"2","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.314272942Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"client":"1","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.314652667Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"client":"2","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.314679433Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"client":"2","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.314694251Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.314750662Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"client":"1","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.314779848Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"client":"1","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.314794986Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.314821402Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"client":"0","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.31483702Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"client":"0","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"[0]R flush_numbers[0..12]: [31, 30, 30, 33, 35, 35, 38, 29, 33, 38, 38, 36]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.314859317Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"":"downstairs","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"[0]R generation[0..12]: [21, 20, 20, 22, 23, 23, 25, 19, 22, 25, 25, 24]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.314874554Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"":"downstairs","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.314888972Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"":"downstairs","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"[1]R flush_numbers[0..12]: [31, 30, 30, 33, 35, 35, 38, 29, 33, 38, 38, 36]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.31490294Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"":"downstairs","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"[1]R generation[0..12]: [21, 20, 20, 22, 23, 23, 25, 19, 22, 25, 25, 24]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.314916768Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"":"downstairs","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.314930926Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"":"downstairs","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"[2]R flush_numbers[0..12]: [31, 30, 30, 33, 35, 35, 38, 29, 33, 38, 38, 36]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.314948004Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"":"downstairs","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"[2]R generation[0..12]: [21, 20, 20, 22, 23, 23, 25, 19, 22, 25, 25, 24]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.314962941Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"":"downstairs","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.314977069Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"":"downstairs","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.314990207Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"":"downstairs","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.315003416Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"":"downstairs","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"Next flush: 39","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.315016503Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"":"downstairs","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.315029442Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"":"downstairs","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"No downstairs reconciliation required"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.315051169Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.315063567Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"client":"0","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.315076855Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"client":"1","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.315090313Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"client":"2","":"downstairs","session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.315103531Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.315115869Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"d591e42c-a432-4af8-8584-a1eecb3f91c3 is now active with session: 13fd48fa-b73c-42ea-b6f2-083416b8fbba","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.315128727Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:41.315140796Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291,"session_id":"13fd48fa-b73c-42ea-b6f2-083416b8fbba"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-03T08:31:41.315153303Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 372 len 24576 data: 6 6 4 4 3 3 02/30 Read block 132 len 24576 03/30 Flush 04/30 Write block 512 len 20480 data: 2 2 2 2 2 05/30 Write block 127 len 4096 data: 4 06/30 Read block 415 len 32768 07/30 Read block 205 len 20480 08/30 Read block 336 len 12288 09/30 Read block 232 len 8192 10/30 Flush 11/30 Write block 143 len 36864 data: 5 5 4 4 5 3 3 3 3 12/30 Write block 541 len 32768 data: 2 2 3 3 3 3 3 3 13/30 Read block 247 len 20480 14/30 Write block 469 len 32768 data: 3 3 3 3 3 3 3 3 15/30 Read block 253 len 24576 16/30 Read block 336 len 36864 17/30 Write block 250 len 36864 data: 5 4 6 5 5 5 5 5 4 18/30 Write block 43 len 32768 data: 2 2 3 3 3 3 4 4 19/30 Read block 93 len 12288 20/30 Read block 2 len 40960 21/30 Read block 131 len 28672 22/30 Read block 234 len 4096 23/30 Write block 241 len 8192 data: 4 4 24/30 Read block 156 len 12288 25/30 Read block 535 len 4096 26/30 Write block 442 len 24576 data: 3 2 2 2 3 3 27/30 Read block 545 len 4096 28/30 Read block 445 len 24576 29/30 Write block 410 len 40960 data: 3 5 6 6 8 7 6 6 6 6 30/30 Read block 134 len 36864 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 9 Done Done Done false 1011 Acked Write 8 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 10 Done Done Done false 1029 Acked Read 9 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 20 20 19 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1009 1009 1009 Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:31:41.335366442Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1291} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 31 31 31 F F F 1 020-039 20 20 20 30 30 30 F F F 2 040-059 20 20 20 30 30 30 T T T 3 060-079 22 22 22 33 33 33 F F F 4 080-099 23 23 23 35 35 35 F F F 5 100-119 23 23 23 35 35 35 F F F 6 120-139 26 26 26 40 40 40 F F F 7 140-159 19 19 19 29 29 29 T T T 8 160-179 22 22 22 33 33 33 F F F 9 180-199 25 25 25 38 38 38 F F F 10 200-219 25 25 25 38 38 38 F F F 11 220-239 24 24 24 36 36 36 F F F 12 240-259 19 19 19 29 29 29 T T T 13 260-279 19 19 19 29 29 29 F F F 14 280-299 23 23 23 35 35 35 F F F 15 300-319 17 17 17 27 27 27 F F F 16 320-339 25 25 25 38 38 38 F F F 17 340-359 22 22 22 33 33 33 F F F 18 360-379 26 26 26 39 39 39 F F F 19 380-399 20 20 20 30 30 30 F F F 20 400-419 20 20 20 30 30 30 T T T 21 420-439 22 22 22 34 34 34 F F F 22 440-459 22 22 22 32 32 32 T T T 23 460-479 23 23 23 35 35 35 T T T 24 480-499 22 22 22 32 32 32 F F F 25 500-519 26 26 26 40 40 40 F F F 26 520-539 25 25 25 38 38 38 F F F 27 540-559 20 20 20 30 30 30 T T T 28 560-579 24 24 24 36 36 36 F F F 29 580-599 24 24 24 36 36 36 F F F Max gen: 26, Max flush: 40 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 31 31 31 F F F 1 020-039 20 20 20 30 30 30 F F F 2 040-059 27 27 27 41 41 41 F F F 3 060-079 22 22 22 33 33 33 F F F 4 080-099 23 23 23 35 35 35 F F F 5 100-119 23 23 23 35 35 35 F F F 6 120-139 26 26 26 40 40 40 F F F 7 140-159 27 27 27 41 41 41 F F F 8 160-179 22 22 22 33 33 33 F F F 9 180-199 25 25 25 38 38 38 F F F 10 200-219 25 25 25 38 38 38 F F F 11 220-239 24 24 24 36 36 36 F F F 12 240-259 27 27 27 41 41 41 F F F 13 260-279 19 19 19 29 29 29 F F F 14 280-299 23 23 23 35 35 35 F F F 15 300-319 17 17 17 27 27 27 F F F 16 320-339 25 25 25 38 38 38 F F F 17 340-359 22 22 22 33 33 33 F F F 18 360-379 26 26 26 39 39 39 F F F 19 380-399 20 20 20 30 30 30 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 22 22 22 34 34 34 F F F 22 440-459 27 27 27 41 41 41 F F F 23 460-479 27 27 27 41 41 41 F F F 24 480-499 22 22 22 32 32 32 F F F 25 500-519 26 26 26 40 40 40 F F F 26 520-539 25 25 25 38 38 38 F F F 27 540-559 27 27 27 41 41 41 F F F 28 560-579 24 24 24 36 36 36 F F F 29 580-599 24 24 24 36 36 36 F F F Max gen: 27, Max flush: 41 Begin loop 14 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.612591236Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.656340069Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.656974236Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.657009771Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.657029958Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.657045486Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299} {"msg":"Crucible a7ee0644-c937-4be1-945c-15873567f091 has session id: 0e5f435a-ac3f-47d7-a188-73b3a70bc7d0","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.657433949Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"Upstairs opts: Upstairs UUID: a7ee0644-c937-4be1-945c-15873567f091, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.657479822Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"Crucible stats registered with UUID: a7ee0644-c937-4be1-945c-15873567f091","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.657506428Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.688268737Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.688300162Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299} {"msg":"a7ee0644-c937-4be1-945c-15873567f091 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.688333007Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.688350405Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"client":"0","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.688376991Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"client":"0","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.688394658Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"client":"1","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.688411656Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"client":"1","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.688428423Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"client":"2","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.688445471Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"client":"2","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.688462068Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"":"io task","client":"2","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.688492284Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"":"io task","client":"0","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.688526559Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"":"io task","client":"1","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:54387)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.688560954Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"":"io task","client":"0","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:39065)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.688596339Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"":"io task","client":"2","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:62211)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.688638553Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"":"io task","client":"1","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.689429007Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"client":"1","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.689458083Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"client":"1","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.689473791Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"client":"0","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.689493778Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"client":"0","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.689534242Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"client":"2","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.689565057Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"client":"2","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.690068034Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"client":"1","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.690262126Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"client":"2","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.690411184Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"client":"0","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.690715259Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"client":"1","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.690743995Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"client":"1","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.690760213Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.690789019Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"client":"2","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.690804576Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"client":"2","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.690818124Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.690883285Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"client":"0","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.690903122Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"client":"0","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"[0]R flush_numbers[0..12]: [31, 30, 41, 33, 35, 35, 40, 41, 33, 38, 38, 36]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.69091795Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"":"downstairs","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"[0]R generation[0..12]: [21, 20, 27, 22, 23, 23, 26, 27, 22, 25, 25, 24]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.690932048Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"":"downstairs","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.690946286Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"":"downstairs","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"[1]R flush_numbers[0..12]: [31, 30, 41, 33, 35, 35, 40, 41, 33, 38, 38, 36]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.690960194Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"":"downstairs","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"[1]R generation[0..12]: [21, 20, 27, 22, 23, 23, 26, 27, 22, 25, 25, 24]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.690974082Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"":"downstairs","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.690988Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"":"downstairs","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"[2]R flush_numbers[0..12]: [31, 30, 41, 33, 35, 35, 40, 41, 33, 38, 38, 36]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.691001858Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"":"downstairs","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"[2]R generation[0..12]: [21, 20, 27, 22, 23, 23, 26, 27, 22, 25, 25, 24]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.691022944Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"":"downstairs","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.69105428Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"":"downstairs","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.691089325Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"":"downstairs","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.691118261Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"":"downstairs","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"Next flush: 42","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.691144257Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"":"downstairs","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.691171133Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"":"downstairs","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.691197699Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.691230174Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"client":"0","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.69125501Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"client":"1","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.691270078Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"client":"2","":"downstairs","session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.691287786Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-03T08:31:45.691310942Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"a7ee0644-c937-4be1-945c-15873567f091 is now active with session: 0e5f435a-ac3f-47d7-a188-73b3a70bc7d0","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.691334299Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.691355856Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299,"session_id":"0e5f435a-ac3f-47d7-a188-73b3a70bc7d0"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:45.691378663Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 296 len 8192 data: 3 4 02/30 Write block 475 len 20480 data: 4 4 3 3 3 03/30 Read block 170 len 40960 04/30 Write block 528 len 32768 data: 8 7 5 5 5 5 4 3 05/30 Read block 473 len 28672 06/30 Read block 273 len 28672 07/30 Read block 3 len 32768 08/30 Read block 482 len 36864 09/30 Read block 479 len 32768 10/30 Write block 445 len 20480 data: 3 4 4 4 4 11/30 Write block 459 len 16384 data: 4 4 4 4 12/30 Read block 433 len 36864 13/30 Read block 65 len 8192 14/30 Read block 414 len 12288 15/30 Write block 425 len 32768 data: 4 3 4 4 4 3 3 3 16/30 Read block 137 len 20480 17/30 Read block 314 len 8192 18/30 Write block 544 len 40960 data: 4 4 4 4 4 3 4 5 4 3 19/30 Write block 304 len 40960 data: 3 3 2 2 2 2 2 2 2 3 20/30 Read block 13 len 4096 21/30 Read block 214 len 24576 22/30 Read block 260 len 24576 23/30 Read block 500 len 4096 24/30 Read block 407 len 28672 25/30 Read block 555 len 4096 26/30 Write block 37 len 24576 data: 2 2 3 2 2 2 27/30 Read block 60 len 20480 28/30 Read block 37 len 8192 29/30 Read block 308 len 40960 30/30 Read block 234 len 36864 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 5 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Write 8 Done Done Done false 1004 Acked Read 7 Done Done Done false 1005 Acked Read 7 Done Done Done false 1006 Acked Read 8 Done Done Done false 1007 Acked Read 9 Done Done Done false 1008 Acked Read 8 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Write 4 Done Done Done false 1011 Acked Read 9 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Write 8 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:31:45.711277257Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1299} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 31 31 31 F F F 1 020-039 20 20 20 30 30 30 T T T 2 040-059 27 27 27 41 41 41 T T T 3 060-079 22 22 22 33 33 33 F F F 4 080-099 23 23 23 35 35 35 F F F 5 100-119 23 23 23 35 35 35 F F F 6 120-139 26 26 26 40 40 40 F F F 7 140-159 27 27 27 41 41 41 F F F 8 160-179 22 22 22 33 33 33 F F F 9 180-199 25 25 25 38 38 38 F F F 10 200-219 25 25 25 38 38 38 F F F 11 220-239 24 24 24 36 36 36 F F F 12 240-259 27 27 27 41 41 41 F F F 13 260-279 19 19 19 29 29 29 F F F 14 280-299 23 23 23 35 35 35 T T T 15 300-319 17 17 17 27 27 27 T T T 16 320-339 25 25 25 38 38 38 F F F 17 340-359 22 22 22 33 33 33 F F F 18 360-379 26 26 26 39 39 39 F F F 19 380-399 20 20 20 30 30 30 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 22 22 22 34 34 34 T T T 22 440-459 27 27 27 41 41 41 T T T 23 460-479 27 27 27 41 41 41 T T T 24 480-499 22 22 22 32 32 32 F F F 25 500-519 26 26 26 40 40 40 F F F 26 520-539 25 25 25 38 38 38 T T T 27 540-559 27 27 27 41 41 41 T T T 28 560-579 24 24 24 36 36 36 F F F 29 580-599 24 24 24 36 36 36 F F F Max gen: 27, Max flush: 41 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 31 31 31 F F F 1 020-039 28 28 28 42 42 42 F F F 2 040-059 28 28 28 42 42 42 F F F 3 060-079 22 22 22 33 33 33 F F F 4 080-099 23 23 23 35 35 35 F F F 5 100-119 23 23 23 35 35 35 F F F 6 120-139 26 26 26 40 40 40 F F F 7 140-159 27 27 27 41 41 41 F F F 8 160-179 22 22 22 33 33 33 F F F 9 180-199 25 25 25 38 38 38 F F F 10 200-219 25 25 25 38 38 38 F F F 11 220-239 24 24 24 36 36 36 F F F 12 240-259 27 27 27 41 41 41 F F F 13 260-279 19 19 19 29 29 29 F F F 14 280-299 28 28 28 42 42 42 F F F 15 300-319 28 28 28 42 42 42 F F F 16 320-339 25 25 25 38 38 38 F F F 17 340-359 22 22 22 33 33 33 F F F 18 360-379 26 26 26 39 39 39 F F F 19 380-399 20 20 20 30 30 30 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 28 28 28 42 42 42 F F F 22 440-459 28 28 28 42 42 42 F F F 23 460-479 28 28 28 42 42 42 F F F 24 480-499 22 22 22 32 32 32 F F F 25 500-519 26 26 26 40 40 40 F F F 26 520-539 28 28 28 42 42 42 F F F 27 540-559 28 28 28 42 42 42 F F F 28 560-579 24 24 24 36 36 36 F F F 29 580-599 24 24 24 36 36 36 F F F Max gen: 28, Max flush: 42 Begin loop 15 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:49.994714875Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:31:50.037600383Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.037971679Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.069202129Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.069773596Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.06981352Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.069828457Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.069843305Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307} {"msg":"Crucible cf4bcf26-bc2c-4be4-8a7a-f3ebd30d33b2 has session id: 57ba9fd5-ff3b-4156-a19d-9e3f96e767ca","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.070274942Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"Upstairs opts: Upstairs UUID: cf4bcf26-bc2c-4be4-8a7a-f3ebd30d33b2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.070327085Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"Crucible stats registered with UUID: cf4bcf26-bc2c-4be4-8a7a-f3ebd30d33b2","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.070354001Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.100608783Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.100654826Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307} {"msg":"cf4bcf26-bc2c-4be4-8a7a-f3ebd30d33b2 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.100688331Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.100719397Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"client":"0","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.10076694Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"client":"0","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.100805165Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"client":"1","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.100841899Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"client":"1","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.100876384Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"client":"2","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.100913039Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"client":"2","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.100945644Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"":"io task","client":"2","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.100977879Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"":"io task","client":"0","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.101010875Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"":"io task","client":"1","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:36829)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.10104123Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"":"io task","client":"2","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:58291)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.101212745Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"":"io task","client":"1","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:36543)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.101230322Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"":"io task","client":"0","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.101860131Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"client":"2","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.101900834Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"client":"2","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.101965395Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"client":"0","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.102009439Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"client":"0","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.102026836Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"client":"1","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.102040894Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"client":"1","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.102491748Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"client":"1","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.102639187Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"client":"2","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.102672882Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"client":"0","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.103055766Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"client":"1","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.10309871Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"client":"1","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.103124576Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.103249737Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"client":"2","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.103277584Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"client":"2","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.103292041Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.103304889Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"client":"0","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.103318598Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"client":"0","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"[0]R flush_numbers[0..12]: [31, 42, 42, 33, 35, 35, 40, 41, 33, 38, 38, 36]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.103332685Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"":"downstairs","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"[0]R generation[0..12]: [21, 28, 28, 22, 23, 23, 26, 27, 22, 25, 25, 24]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.103346433Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"":"downstairs","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.103360521Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"":"downstairs","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"[1]R flush_numbers[0..12]: [31, 42, 42, 33, 35, 35, 40, 41, 33, 38, 38, 36]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.103374469Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"":"downstairs","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"[1]R generation[0..12]: [21, 28, 28, 22, 23, 23, 26, 27, 22, 25, 25, 24]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.103388327Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"":"downstairs","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.103402375Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"":"downstairs","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"[2]R flush_numbers[0..12]: [31, 42, 42, 33, 35, 35, 40, 41, 33, 38, 38, 36]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.103419503Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"":"downstairs","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"[2]R generation[0..12]: [21, 28, 28, 22, 23, 23, 26, 27, 22, 25, 25, 24]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.103433901Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"":"downstairs","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.103455478Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"":"downstairs","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"Max found gen is 29","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.103474885Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"":"downstairs","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"Generation requested: 30 >= found:29","v":0,"name":"crucible","level":30,"timeWait for a query_work_queue command to finish before sending IO ":"2025-01-03T08:31:50.103497051Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"":"downstairs","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"Next flush: 43","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.103523788Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"":"downstairs","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.103537615Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"":"downstairs","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.103550864Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.103563222Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"client":"0","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.10357728Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"client":"1","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.103590768Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"client":"2","":"downstairs","session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.103604136Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.103616374Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"cf4bcf26-bc2c-4be4-8a7a-f3ebd30d33b2 is now active with session: 57ba9fd5-ff3b-4156-a19d-9e3f96e767ca","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.103629332Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.10364151Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307,"session_id":"57ba9fd5-ff3b-4156-a19d-9e3f96e767ca"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:50.103656678Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 240 len 24576 data: 4 5 5 4 5 5 02/30 Read block 408 len 4096 03/30 Read block 320 len 36864 04/30 Read block 140 len 40960 05/30 Write block 221 len 16384 data: 4 4 6 5 06/30 Write block 85 len 8192 data: 4 3 07/30 Read block 427 len 20480 08/30 Read block 428 len 8192 09/30 Write block 183 len 36864 data: 5 6 6 6 6 6 5 5 4 10/30 Write block 29 len 8192 data: 3 2 11/30 Read block 37 len 32768 12/30 Write block 162 len 40960 data: 6 6 6 6 5 4 2 3 3 3 13/30 Write block 567 len 12288 data: 3 3 3 14/30 Write block 72 len 36864 data: 3 3 3 5 4 3 3 3 3 15/30 Read block 262 len 8192 16/30 Write block 339 len 20480 data: 4 4 4 4 3 17/30 Read block 522 len 40960 18/30 Read block 46 len 36864 19/30 Read block 532 len 8192 20/30 Read block 147 len 28672 21/30 Write block 552 len 8192 data: 5 4 22/30 Read block 182 len 12288 23/30 Read block 246 len 16384 24/30 Read block 380 len 8192 25/30 Read block 107 len 12288 26/30 Write block 98 len 8192 data: 5 7 27/30 Write block 239 len 40960 data: 3 5 6 6 5 6 6 4 5 5 28/30 Write block 230 len 40960 data: 4 4 4 5 5 4 4 2 3 4 29/30 Write block 513 len 24576 data: 3 3 3 3 2 2 30/30 Read block 95 len 24576 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Read 9 Done Done Done false 1003 Acked Read 10 Done Done Done false 1004 Acked Write 4 Done Done Done false 1005 Acked Write 2 Done Done Done false 1006 Acked Read 5 Done Done Done false 1007 Acked Read 2 Done Done Done false 1008 Acked Write 9 Done Done Done false 1009 Acked Write 2 Done Done Done false 1010 Acked Read 8 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Write 3 Done Done Done false 1013 Acked Write 9 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Write 5 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 6 Done Done Sent false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 29 30 28 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:31:50.122667502Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1307} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 31 31 31 F F F 1 020-039 28 28 28 42 42 42 T T T 2 040-059 28 28 28 42 42 42 F F F 3 060-079 22 22 22 33 33 33 T T T 4 080-099 23 23 23 35 35 35 T T T 5 100-119 23 23 23 35 35 35 F F F 6 120-139 26 26 26 40 40 40 F F F 7 140-159 27 27 27 41 41 41 F F F 8 160-179 22 22 22 33 33 33 T T T 9 180-199 25 25 25 38 38 38 T T T 10 200-219 25 25 25 38 38 38 F F F 11 220-239 24 24 24 36 36 36 T T T 12 240-259 27 27 27 41 41 41 T T T 13 260-279 19 19 19 29 29 29 F F F 14 280-299 28 28 28 42 42 42 F F F 15 300-319 28 28 28 42 42 42 F F F 16 320-339 25 25 25 38 38 38 T T T 17 340-359 22 22 22 33 33 33 T T T 18 360-379 26 26 26 39 39 39 F F F 19 380-399 20 20 20 30 30 30 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 28 28 28 42 42 42 F F F 22 440-459 28 28 28 42 42 42 F F F 23 460-479 28 28 28 42 42 42 F F F 24 480-499 22 22 22 32 32 32 F F F 25 500-519 26 26 26 40 40 40 T T T 26 520-539 28 28 28 42 42 42 F F F 27 540-559 28 28 28 42 42 42 T T T 28 560-579 24 24 24 36 36 36 T T T 29 580-599 24 24 24 36 36 36 F F F Max gen: 28, Max flush: 42 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 31 31 31 F F F 1 020-039 29 29 29 43 43 43 F F F 2 040-059 28 28 28 42 42 42 F F F 3 060-079 29 29 29 43 43 43 F F F 4 080-099 29 29 29 43 43 43 F F F 5 100-119 23 23 23 35 35 35 F F F 6 120-139 26 26 26 40 40 40 F F F 7 140-159 27 27 27 41 41 41 F F F 8 160-179 29 29 29 43 43 43 F F F 9 180-199 29 29 29 43 43 43 F F F 10 200-219 25 25 25 38 38 38 F F F 11 220-239 29 29 29 43 43 43 F F F 12 240-259 29 29 29 43 43 43 F F F 13 260-279 19 19 19 29 29 29 F F F 14 280-299 28 28 28 42 42 42 F F F 15 300-319 28 28 28 42 42 42 F F F 16 320-339 29 29 29 43 43 43 F F F 17 340-359 29 29 29 43 43 43 F F F 18 360-379 26 26 26 39 39 39 F F F 19 380-399 20 20 20 30 30 30 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 28 28 28 42 42 42 F F F 22 440-459 28 28 28 42 42 42 F F F 23 460-479 28 28 28 42 42 42 F F F 24 480-499 22 22 22 32 32 32 F F F 25 500-519 29 29 29 43 43 43 F F F 26 520-539 28 28 28 42 42 42 F F F 27 540-559 29 29 29 43 43 43 F F F 28 560-579 29 29 29 43 43 43 F F F 29 580-599 24 24 24 36 36 36 F F F Max gen: 29, Max flush: 43 Begin loop 16 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.456156042Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.500282499Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.501052227Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.501090931Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.501105879Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.501121406Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315} {"msg":"Crucible f70e5e84-56f4-4351-91b3-710fdfec77be has session id: 6b7493cb-ce85-4c7c-8986-c78d75b7e738","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.501716859Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"Upstairs opts: Upstairs UUID: f70e5e84-56f4-4351-91b3-710fdfec77be, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.501757084Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"Crucible stats registered with UUID: f70e5e84-56f4-4351-91b3-710fdfec77be","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.501774281Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.532217776Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.532250081Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315} {"msg":"f70e5e84-56f4-4351-91b3-710fdfec77be active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.532284186Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.532301234Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"client":"0","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.53232798Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"client":"0","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.532345977Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"client":"1","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.532363225Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"client":"1","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.532390761Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"client":"2","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.532425506Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"client":"2","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.532452822Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"":"io task","client":"2","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.532479868Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"":"io task","client":"0","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.532508823Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"":"io task","client":"1","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"ds_connection connected from Ok(127.0.0.1:36520)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.532538509Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"":"io task","client":"2","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"ds_connection connected from Ok(127.0.0.1:35373)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.53259959Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"":"io task","client":"0","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"ds_connection connected from Ok(127.0.0.1:40502)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.532644174Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"":"io task","client":"1","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.533327214Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"client":"1","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.53335655Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"client":"1","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.533537183Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"client":"0","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.533581307Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"client":"0","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.533613812Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"client":"2","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.533642648Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"client":"2","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.533942684Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"client":"1","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.534081874Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"client":"0","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.534364503Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"client":"2","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.534489414Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"client":"1","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.53451943Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"client":"1","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.534535467Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.534561914Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"client":"0","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.534576372Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"client":"0","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.53458977Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.534854931Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"client":"2","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.534874728Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"client":"2","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"[0]R flush_numbers[0..12]: [31, 43, 42, 43, 43, 35, 40, 41, 43, 43, 38, 43]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.534889786Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"":"downstairs","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"[0]R generation[0..12]: [21, 29, 28, 29, 29, 23, 26, 27, 29, 29, 25, 29]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.534903974Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"":"downstairs","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.534918122Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"":"downstairs","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"[1]R flush_numbers[0..12]: [31, 43, 42, 43, 43, 35, 40, 41, 43, 43, 38, 43]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.53493206Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"":"downstairs","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"[1]R generation[0..12]: [21, 29, 28, 29, 29, 23, 26, 27, 29, 29, 25, 29]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.534945708Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"":"downstairs","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.534963045Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"":"downstairs","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"[2]R flush_numbers[0..12]: [31, 43, 42, 43, 43, 35, 40, 41, 43, 43, 38, 43]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.534977193Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"":"downstairs","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"[2]R generation[0..12]: [21, 29, 28, 29, 29, 23, 26, 27, 29, 29, 25, 29]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.534990811Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"":"downstairs","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.535004919Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"":"downstairs","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.535018147Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"":"downstairs","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"Generation requested: 32 >= found:30","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.535031635Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"":"downstairs","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"Next flush: 44","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.535044803Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"":"downstairs","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.535057921Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid"Wait for a query_work_queue command to finish before sending IO :1315,"":"downstairs","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.535081098Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.535093636Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"client":"0","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.535107234Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"client":"1","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.535120512Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"client":"2","":"downstairs","session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.53513387Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.535146288Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"f70e5e84-56f4-4351-91b3-710fdfec77be is now active with session: 6b7493cb-ce85-4c7c-8986-c78d75b7e738","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.535159236Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:54.535171415Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315,"session_id":"6b7493cb-ce85-4c7c-8986-c78d75b7e738"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-03T08:31:54.535186103Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 579 len 36864 data: 6 6 5 5 5 5 4 4 4 02/30 Read block 415 len 16384 03/30 Read block 417 len 16384 04/30 Write block 234 len 16384 data: 6 5 5 3 05/30 Write block 490 len 16384 data: 2 2 2 2 06/30 Write block 46 len 40960 data: 4 4 4 5 5 4 5 3 3 2 07/30 Read block 67 len 36864 08/30 Read block 573 len 32768 09/30 Write block 306 len 28672 data: 3 3 3 3 3 3 3 10/30 Flush 11/30 Write block 529 len 40960 data: 8 6 6 6 6 5 4 3 3 3 12/30 Flush 13/30 Write block 584 len 4096 data: 6 14/30 Flush 15/30 Write block 368 len 20480 data: 5 5 6 6 7 16/30 Read block 368 len 16384 17/30 Read block 105 len 8192 18/30 Write block 101 len 36864 data: 4 5 6 5 5 7 7 6 5 19/30 Write block 146 len 4096 data: 5 20/30 Read block 586 len 16384 21/30 Read block 316 len 12288 22/30 Read block 177 len 20480 23/30 Write block 439 len 28672 data: 4 3 3 4 3 3 4 24/30 Write block 419 len 40960 data: 7 5 5 3 3 3 5 4 5 5 25/30 Read block 525 len 8192 26/30 Read block 587 len 36864 27/30 Read block 25 len 12288 28/30 Read block 316 len 4096 29/30 Read block 545 len 20480 30/30 Read block 73 len 24576 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 5 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 6 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 15 16 16 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:31:54.555892949Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1315} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 31 31 31 F F F 1 020-039 29 29 29 43 43 43 F F F 2 040-059 32 32 32 44 44 44 F F F 3 060-079 29 29 29 43 43 43 F F F 4 080-099 29 29 29 43 43 43 F F F 5 100-119 23 23 23 35 35 35 T T T 6 120-139 26 26 26 40 40 40 F F F 7 140-159 27 27 27 41 41 41 T T T 8 160-179 29 29 29 43 43 43 F F F 9 180-199 29 29 29 43 43 43 F F F 10 200-219 25 25 25 38 38 38 F F F 11 220-239 32 32 32 44 44 44 F F F 12 240-259 29 29 29 43 43 43 F F F 13 260-279 19 19 19 29 29 29 F F F 14 280-299 28 28 28 42 42 42 F F F 15 300-319 32 32 32 44 44 44 F F F 16 320-339 29 29 29 43 43 43 F F F 17 340-359 29 29 29 43 43 43 F F F 18 360-379 26 26 26 39 39 39 T T T 19 380-399 20 20 20 30 30 30 F F F 20 400-419 27 27 27 41 41 41 T T T 21 420-439 28 28 28 42 42 42 T T T 22 440-459 28 28 28 42 42 42 T T T 23 460-479 28 28 28 42 42 42 F F F 24 480-499 32 32 32 44 44 44 F F F 25 500-519 29 29 29 43 43 43 F F F 26 520-539 32 32 32 45 45 45 F F F 27 540-559 29 29 29 43 43 43 F F F 28 560-579 32 32 32 44 44 44 F F F 29 580-599 32 32 32 46 46 46 F F F Max gen: 32, Max flush: 46 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 31 31 31 F F F 1 020-039 29 29 29 43 43 43 F F F 2 040-059 32 32 32 44 44 44 F F F 3 060-079 29 29 29 43 43 43 F F F 4 080-099 29 29 29 43 43 43 F F F 5 100-119 33 33 33 47 47 47 F F F 6 120-139 26 26 26 40 40 40 F F F 7 140-159 33 33 33 47 47 47 F F F 8 160-179 29 29 29 43 43 43 F F F 9 180-199 29 29 29 43 43 43 F F F 10 200-219 25 25 25 38 38 38 F F F 11 220-239 32 32 32 44 44 44 F F F 12 240-259 29 29 29 43 43 43 F F F 13 260-279 19 19 19 29 29 29 F F F 14 280-299 28 28 28 42 42 42 F F F 15 300-319 32 32 32 44 44 44 F F F 16 320-339 29 29 29 43 43 43 F F F 17 340-359 29 29 29 43 43 43 F F F 18 360-379 33 33 33 47 47 47 F F F 19 380-399 20 20 20 30 30 30 F F F 20 400-419 33 33 33 47 47 47 F F F 21 420-439 33 33 33 47 47 47 F F F 22 440-459 33 33 33 47 47 47 F F F 23 460-479 28 28 28 42 42 42 F F F 24 480-499 32 32 32 44 44 44 F F F 25 500-519 29 29 29 43 43 43 F F F 26 520-539 32 32 32 45 45 45 F F F 27 540-559 29 29 29 43 43 43 F F F 28 560-579 32 32 32 44 44 44 F F F 29 580-599 32 32 32 46 46 46 F F F Max gen: 33, Max flush: 47 Begin loop 17 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.827665999Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.871418761Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.872079734Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.872123298Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.872145435Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.872163722Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323} {"msg":"Crucible 3e5b2d43-a089-4b3c-9411-5fdcf2b5fc9d has session id: 06da45d8-817c-4ccc-8c42-36dea6b17530","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.872621675Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"Upstairs opts: Upstairs UUID: 3e5b2d43-a089-4b3c-9411-5fdcf2b5fc9d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.872677587Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"Crucible stats registered with UUID: 3e5b2d43-a089-4b3c-9411-5fdcf2b5fc9d","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.872699494Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.903327112Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.903363856Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323} {"msg":"3e5b2d43-a089-4b3c-9411-5fdcf2b5fc9d active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.903397691Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.903414759Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"client":"0","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.903443005Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"client":"0","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.903460602Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"client":"1","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.90347766Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"client":"1","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.903494437Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"client":"2","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.903511195Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"client":"2","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.903597412Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"":"io task","client":"2","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.903644326Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"":"io task","client":"0","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.903678321Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"":"io task","client":"1","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"ds_connection connected from Ok(127.0.0.1:59411)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.903711916Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"":"io task","client":"2","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"ds_connection connected from Ok(127.0.0.1:43861)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.903746061Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"":"io task","client":"1","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"ds_connection connected from Ok(127.0.0.1:61705)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.903781746Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"":"io task","client":"0","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.90436844Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"client":"0","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.904396936Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"client":"0","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.904422922Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"client":"1","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.904452168Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"client":"1","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.904611314Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"client":"2","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.90464298Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"client":"2","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.905102503Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"client":"1","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.905145216Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"client":"0","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.905179162Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"client":"2","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.905633775Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"client":"1","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.905662401Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"client":"1","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.905679109Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.905752688Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"client":"0","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.905780474Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"client":"0","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.905795552Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.905809059Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"client":"2","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.905822657Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"client":"2","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"[0]R flush_numbers[0..12]: [31, 43, 44, 43, 43, 47, 40, 47, 43, 43, 38, 44]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.905836675Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"":"downstairs","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"[0]R generation[0..12]: [21, 29, 32, 29, 29, 33, 26, 33, 29, 29, 25, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.905850973Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"":"downstairs","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.905865151Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"":"downstairs","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"[1]R flush_numbers[0..12]: [31, 43, 44, 43, 43, 47, 40, 47, 43, 43, 38, 44]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.905878929Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"":"downstairs","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"[1]R generation[0..12]: [21, 29, 32, 29, 29, 33, 26, 33, 29, 29, 25, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.905892697Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"":"downstairs","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.905906875Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"":"downstairs","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"[2]R flush_numbers[0..12]: [31, 43, 44, 43, 43, 47, 40, 47, 43, 43, 38, 44]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.905920843Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"":"downstairs","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"[2]R generation[0..12]: [21, 29, 32, 29, 29, 33, 26, 33, 29, 29, 25, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.905938231Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"":"downstairs","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.905952718Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"":"downstairs","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.905965867Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"":"downstairs","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.905979185Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"":"downstairs","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"Wait for a query_work_queue command to finish before sending IO msg":"Next flush: 48","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.90601476Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"":"downstairs","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.906030737Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"":"downstairs","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.906044415Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.906056583Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"client":"0","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.906069791Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"client":"1","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.906083039Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"client":"2","":"downstairs","session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.906096297Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.906108646Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"3e5b2d43-a089-4b3c-9411-5fdcf2b5fc9d is now active with session: 06da45d8-817c-4ccc-8c42-36dea6b17530","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.906121494Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.906133752Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323,"session_id":"06da45d8-817c-4ccc-8c42-36dea6b17530"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:31:58.90614634Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 84 len 8192 data: 3 5 02/30 Write block 227 len 16384 data: 3 4 4 5 03/30 Write block 70 len 16384 data: 4 4 4 4 04/30 Write block 125 len 16384 data: 6 5 5 3 05/30 Read block 177 len 36864 06/30 Write block 239 len 40960 data: 5 6 7 7 6 7 7 5 6 6 07/30 Write block 491 len 12288 data: 3 3 3 08/30 Read block 356 len 4096 09/30 Write block 33 len 40960 data: 2 2 2 2 3 3 4 3 3 3 10/30 Read block 32 len 20480 11/30 Read block 254 len 8192 12/30 Read block 119 len 8192 13/30 Write block 271 len 4096 data: 3 14/30 Read block 171 len 20480 15/30 Write block 542 len 36864 data: 3 4 5 5 5 5 5 4 5 16/30 Write block 252 len 36864 data: 7 6 6 6 6 6 5 5 4 17/30 Write block 552 len 32768 data: 6 5 3 3 4 4 4 3 18/30 Read block 119 len 16384 19/30 Write block 74 len 36864 data: 4 6 5 4 4 4 4 4 3 20/30 Read block 519 len 8192 21/30 Flush 22/30 Write block 334 len 36864 data: 3 2 2 2 3 5 5 5 5 23/30 Write block 413 len 20480 data: 7 9 8 7 7 24/30 Read block 126 len 12288 25/30 Read block 219 len 28672 26/30 Write block 549 len 28672 data: 5 6 6 7 6 4 4 27/30 Write block 39 len 32768 data: 5 4 4 4 3 3 4 5 28/30 Write block 512 len 40960 data: 3 4 4 4 4 3 3 2 3 6 29/30 Read block 551 len 24576 30/30 Read block 412 len 4096 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 9 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 1 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 9 8 9 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:31:58.925475708Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1323} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 31 31 31 F F F 1 020-039 34 34 34 48 48 48 T T T 2 040-059 34 34 34 48 48 48 T T T 3 060-079 34 34 34 48 48 48 F F F 4 080-099 34 34 34 48 48 48 F F F 5 100-119 33 33 33 47 47 47 F F F 6 120-139 34 34 34 48 48 48 F F F 7 140-159 33 33 33 47 47 47 F F F 8 160-179 29 29 29 43 43 43 F F F 9 180-199 29 29 29 43 43 43 F F F 10 200-219 25 25 25 38 38 38 F F F 11 220-239 34 34 34 48 48 48 F F F 12 240-259 34 34 34 48 48 48 F F F 13 260-279 34 34 34 48 48 48 F F F 14 280-299 28 28 28 42 42 42 F F F 15 300-319 32 32 32 44 44 44 F F F 16 320-339 29 29 29 43 43 43 T T T 17 340-359 29 29 29 43 43 43 T T T 18 360-379 33 33 33 47 47 47 F F F 19 380-399 20 20 20 30 30 30 F F F 20 400-419 33 33 33 47 47 47 T T T 21 420-439 33 33 33 47 47 47 F F F 22 440-459 33 33 33 47 47 47 F F F 23 460-479 28 28 28 42 42 42 F F F 24 480-499 34 34 34 48 48 48 F F F 25 500-519 29 29 29 43 43 43 T T T 26 520-539 32 32 32 45 45 45 T T T 27 540-559 34 34 34 48 48 48 T T T 28 560-579 32 32 32 44 44 44 F F F 29 580-599 32 32 32 46 46 46 F F F Max gen: 34, Max flush: 48 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 31 31 31 F F F 1 020-039 35 35 35 49 49 49 F F F 2 040-059 35 35 35 49 49 49 F F F 3 060-079 34 34 34 48 48 48 F F F 4 080-099 34 34 34 48 48 48 F F F 5 100-119 33 33 33 47 47 47 F F F 6 120-139 34 34 34 48 48 48 F F F 7 140-159 33 33 33 47 47 47 F F F 8 160-179 29 29 29 43 43 43 F F F 9 180-199 29 29 29 43 43 43 F F F 10 200-219 25 25 25 38 38 38 F F F 11 220-239 34 34 34 48 48 48 F F F 12 240-259 34 34 34 48 48 48 F F F 13 260-279 34 34 34 48 48 48 F F F 14 280-299 28 28 28 42 42 42 F F F 15 300-319 32 32 32 44 44 44 F F F 16 320-339 35 35 35 49 49 49 F F F 17 340-359 35 35 35 49 49 49 F F F 18 360-379 33 33 33 47 47 47 F F F 19 380-399 20 20 20 30 30 30 F F F 20 400-419 35 35 35 49 49 49 F F F 21 420-439 33 33 33 47 47 47 F F F 22 440-459 33 33 33 47 47 47 F F F 23 460-479 28 28 28 42 42 42 F F F 24 480-499 34 34 34 48 48 48 F F F 25 500-519 35 35 35 49 49 49 F F F 26 520-539 35 35 35 49 49 49 F F F 27 540-559 35 35 35 49 49 49 F F F 28 560-579 32 32 32 44 44 44 F F F 29 580-599 32 32 32 46 46 46 F F F Max gen: 35, Max flush: 49 Begin loop 18 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.210427255Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.253711535Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.25428874Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.254325345Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.254340513Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.254355471Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331} {"msg":"Crucible 30a41128-68e3-476c-b2f7-e51fd1da8ae0 has session id: 075919f1-a56e-4477-8e6e-1fec27df0e37","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.254831561Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"Upstairs opts: Upstairs UUID: 30a41128-68e3-476c-b2f7-e51fd1da8ae0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.254879924Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"Crucible stats registered with UUID: 30a41128-68e3-476c-b2f7-e51fd1da8ae0","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.254904941Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.284875745Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.28490837Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331} {"msg":"30a41128-68e3-476c-b2f7-e51fd1da8ae0 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.28497448Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.285007745Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"client":"0","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.28503995Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"client":"0","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.285058508Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"client":"1","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.285076355Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"client":"1","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.285099412Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"client":"2","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.285133147Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"client":"2","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.285166302Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"":"io task","client":"2","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.285203307Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"":"io task","client":"0","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.285237972Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"":"io task","client":"1","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"ds_connection connected from Ok(127.0.0.1:36040)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.285273396Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"":"io task","client":"2","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"ds_connection connected from Ok(127.0.0.1:34124)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.28531648Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"":"io task","client":"0","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"ds_connection connected from Ok(127.0.0.1:47934)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.285457389Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"":"io task","client":"1","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.285816087Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"client":"0","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.2858661Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"client":"0","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.286030586Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"client":"1","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.286060081Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"client":"1","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.286086198Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"client":"2","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.286114224Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"client":"2","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.286527823Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"client":"0","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.286575206Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"client":"1","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.286783096Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"client":"2","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.287089321Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"client":"1","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.287111978Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"client":"1","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.287126386Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.287188587Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"client":"0","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.287212083Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"client":"0","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.287225961Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.287250488Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"client":"2","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.287264546Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"client":"2","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"[0]R flush_numbers[0..12]: [31, 49, 49, 48, 48, 47, 48, 47, 43, 43, 38, 48]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.287290322Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"":"downstairs","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"[0]R generation[0..12]: [21, 35, 35, 34, 34, 33, 34, 33, 29, 29, 25, 34]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.28730457Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"":"downstairs","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.287319078Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"":"downstairs","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"[1]R flush_numbers[0..12]: [31, 49, 49, 48, 48, 47, 48, 47, 43, 43, 38, 48]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.287333016Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"":"downstairs","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"[1]R generation[0..12]: [21, 35, 35, 34, 34, 33, 34, 33, 29, 29, 25, 34]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.287346724Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"":"downstairs","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.287360922Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"":"downstairs","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"[2]R flush_numbers[0..12]: [31, 49, 49, 48, 48, 47, 48, 47, 43, 43, 38, 48]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.28737476Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"":"downstairs","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"[2]R generation[0..12]: [21, 35, 35, 34, 34, 33, 34, 33, 29, 29, 25, 34]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.287389687Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"":"downstairs","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.287415674Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"":"downstairs","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.28744299Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"":"downstairs","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.287468336Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"":"downstairs","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"Next flush: 50","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:32:03.287492982Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"":"downstairs","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.287523518Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"":"downstairs","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.287559693Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.287573591Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"client":"0","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.287587349Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"client":"1","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.287610625Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"client":"2","":"downstairs","session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.287637591Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.287663427Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"30a41128-68e3-476c-b2f7-e51fd1da8ae0 is now active with session: 075919f1-a56e-4477-8e6e-1fec27df0e37","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.287680625Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.287698202Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331,"session_id":"075919f1-a56e-4477-8e6e-1fec27df0e37"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:03.287721519Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 453 len 28672 data: 5 5 5 3 3 4 5 02/30 Write block 189 len 16384 data: 6 6 5 3 03/30 Flush 04/30 Write block 409 len 36864 data: 3 4 6 7 8 10 9 8 8 05/30 Write block 113 len 16384 data: 2 2 3 3 06/30 Read block 460 len 12288 07/30 Write block 176 len 16384 data: 5 4 4 4 08/30 Write block 548 len 20480 data: 6 6 7 7 8 09/30 Read block 533 len 4096 10/30 Read block 242 len 4096 11/30 Write block 258 len 36864 data: 6 6 5 3 3 3 3 3 3 12/30 Write block 493 len 20480 data: 4 3 4 4 4 13/30 Read block 413 len 40960 14/30 Write block 358 len 8192 data: 6 6 15/30 Read block 0 len 4096 16/30 Read block 331 len 28672 17/30 Read block 87 len 40960 18/30 Read block 136 len 36864 19/30 Read block 90 len 4096 20/30 Write block 231 len 36864 data: 5 5 6 7 6 6 4 4 6 21/30 Write block 396 len 20480 data: 5 5 5 5 5 22/30 Write block 119 len 8192 data: 7 7 23/30 Write block 392 len 40960 data: 5 5 5 4 6 6 6 6 6 5 24/30 Write block 376 len 20480 data: 4 4 3 3 4 25/30 Read block 592 len 28672 26/30 Write block 198 len 12288 data: 4 5 5 27/30 Read block 406 len 16384 28/30 Write block 48 len 36864 data: 5 6 6 5 6 4 4 3 4 29/30 Write block 480 len 4096 data: 3 30/30 Write block 588 len 28672 data: 5 4 4 4 4 4 4 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 9 Done Done Done false 1004 Acked Write 4 Done Done Done false 1005 Acked Read 3 Done Done Done false 1006 Acked Write 4 Done Done Done false 1007 Acked Write 5 Done Done Done false 1008 Acked Read 1 Done Done Done false 1009 Acked Read 1 Done Done Done false 1010 Acked Write 9 Done Done Done false 1011 Acked Write 5 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Write 2 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 9 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 24 24 24 72 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1002 1002 1002 Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:32:03.305304412Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1331} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 31 31 31 F F F 1 020-039 35 35 35 49 49 49 F F F 2 040-059 35 35 35 49 49 49 T T T 3 060-079 34 34 34 48 48 48 F F F 4 080-099 34 34 34 48 48 48 F F F 5 100-119 33 33 33 47 47 47 T T T 6 120-139 34 34 34 48 48 48 T T T 7 140-159 33 33 33 47 47 47 F F F 8 160-179 29 29 29 43 43 43 T T T 9 180-199 36 36 36 50 50 50 T T T 10 200-219 25 25 25 38 38 38 T T T 11 220-239 34 34 34 48 48 48 T T T 12 240-259 34 34 34 48 48 48 T T T 13 260-279 34 34 34 48 48 48 T T T 14 280-299 28 28 28 42 42 42 F F F 15 300-319 32 32 32 44 44 44 F F F 16 320-339 35 35 35 49 49 49 F F F 17 340-359 35 35 35 49 49 49 T T T 18 360-379 33 33 33 47 47 47 T T T 19 380-399 20 20 20 30 30 30 T T T 20 400-419 35 35 35 49 49 49 T T T 21 420-439 33 33 33 47 47 47 F F F 22 440-459 36 36 36 50 50 50 F F F 23 460-479 28 28 28 42 42 42 F F F 24 480-499 34 34 34 48 48 48 T T T 25 500-519 35 35 35 49 49 49 F F F 26 520-539 35 35 35 49 49 49 F F F 27 540-559 35 35 35 49 49 49 T T T 28 560-579 32 32 32 44 44 44 F F F 29 580-599 32 32 32 46 46 46 T T T Max gen: 36, Max flush: 50 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 31 31 31 F F F 1 020-039 35 35 35 49 49 49 F F F 2 040-059 37 37 37 51 51 51 F F F 3 060-079 34 34 34 48 48 48 F F F 4 080-099 34 34 34 48 48 48 F F F 5 100-119 37 37 37 51 51 51 F F F 6 120-139 37 37 37 51 51 51 F F F 7 140-159 33 33 33 47 47 47 F F F 8 160-179 37 37 37 51 51 51 F F F 9 180-199 37 37 37 51 51 51 F F F 10 200-219 37 37 37 51 51 51 F F F 11 220-239 37 37 37 51 51 51 F F F 12 240-259 37 37 37 51 51 51 F F F 13 260-279 37 37 37 51 51 51 F F F 14 280-299 28 28 28 42 42 42 F F F 15 300-319 32 32 32 44 44 44 F F F 16 320-339 35 35 35 49 49 49 F F F 17 340-359 37 37 37 51 51 51 F F F 18 360-379 37 37 37 51 51 51 F F F 19 380-399 37 37 37 51 51 51 F F F 20 400-419 37 37 37 51 51 51 F F F 21 420-439 33 33 33 47 47 47 F F F 22 440-459 36 36 36 50 50 50 F F F 23 460-479 28 28 28 42 42 42 F F F 24 480-499 37 37 37 51 51 51 F F F 25 500-519 35 35 35 49 49 49 F F F 26 520-539 35 35 35 49 49 49 F F F 27 540-559 37 37 37 51 51 51 F F F 28 560-579 32 32 32 44 44 44 F F F 29 580-599 37 37 37 51 51 51 F F F Max gen: 37, Max flush: 51 Begin loop 19 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.63312359Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.676479059Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.677049086Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.67708663Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.677102678Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.677118146Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339} {"msg":"Crucible 37ec78e2-991d-48bc-86ee-d838495b5cec has session id: 3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.677531665Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"Upstairs opts: Upstairs UUID: 37ec78e2-991d-48bc-86ee-d838495b5cec, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.677577209Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"Crucible stats registered with UUID: 37ec78e2-991d-48bc-86ee-d838495b5cec","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.677606194Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.707966972Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.708000936Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339} {"msg":"37ec78e2-991d-48bc-86ee-d838495b5cec active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.708052399Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.708072756Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"client":"0","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.708099212Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"client":"0","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.70811721Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"client":"1","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.708135447Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"client":"1","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.708166572Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"client":"2","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.708201747Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"client":"2","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.708235692Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"":"io task","client":"2","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.708272387Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"":"io task","client":"0","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.708306502Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"":"io task","client":"1","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"ds_connection connected from Ok(127.0.0.1:38686)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.708339037Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"":"io task","client":"0","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"ds_connection connected from Ok(127.0.0.1:61273)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.708487445Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"":"io task","client":"2","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"ds_connection connected from Ok(127.0.0.1:62090)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.708537578Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"":"io task","client":"1","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.709040475Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"client":"2","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.709078909Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"client":"2","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.709106775Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"client":"0","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.709136501Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"client":"0","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.7092115Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"client":"1","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.709235276Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"client":"1","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.709733683Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"client":"2","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.709776567Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"client":"0","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.709810202Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"client":"1","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.71030434Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"client":"1","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.710333386Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"client":"1","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.710348484Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.710361362Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"client":"2","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.710379409Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"client":"2","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.710393287Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.710405715Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"client":"0","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.710419114Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"client":"0","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"[0]R flush_numbers[0..12]: [31, 49, 51, 48, 48, 51, 51, 47, 51, 51, 51, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.710432952Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"":"downstairs","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"[0]R generation[0..12]: [21, 35, 37, 34, 34, 37, 37, 33, 37, 37, 37, 37]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.710446879Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"":"downstairs","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.710461007Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"":"downstairs","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"[1]R flush_numbers[0..12]: [31, 49, 51, 48, 48, 51, 51, 47, 51, 51, 51, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.710474825Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"":"downstairs","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"[1]R generation[0..12]: [21, 35, 37, 34, 34, 37, 37, 33, 37, 37, 37, 37]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.710488563Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"":"downstairs","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.710502591Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"":"downstairs","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"[2]R flush_numbers[0..12]: [31, 49, 51, 48, 48, 51, 51, 47, 51, 51, 51, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.710516559Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"":"downstairs","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"[2]R generation[0..12]: [21, 35, 37, 34, 34, 37, 37, 33, 37, 37, 37, 37]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.710530347Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"":"downstairs","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.710544475Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"":"downstairs","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.710557553Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"":"downstairs","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.710572711Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,""Wait for a query_work_queue command to finish before sending IO :"downstairs","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"Next flush: 52","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.710597447Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"":"downstairs","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.710610846Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"":"downstairs","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.710624433Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.710636792Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"client":"0","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.71065022Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"client":"1","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.710663568Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"client":"2","":"downstairs","session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.710676986Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.710689634Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"37ec78e2-991d-48bc-86ee-d838495b5cec is now active with session: 3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.710702612Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.7107148Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339,"session_id":"3f6b257d-012f-45d8-b1c6-9cb5a2c1dbea"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:07.710727349Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 85 len 4096 data: 6 02/30 Write block 352 len 4096 data: 4 03/30 Read block 3 len 36864 04/30 Read block 423 len 32768 05/30 Read block 93 len 40960 06/30 Read block 16 len 16384 07/30 Read block 51 len 36864 08/30 Write block 183 len 36864 data: 6 7 7 7 7 7 7 7 6 09/30 Write block 160 len 40960 data: 6 6 7 7 7 7 6 5 3 4 10/30 Read block 369 len 36864 11/30 Read block 178 len 20480 12/30 Read block 25 len 40960 13/30 Read block 493 len 32768 14/30 Write block 435 len 40960 data: 4 4 4 4 5 4 4 5 4 4 15/30 Read block 163 len 8192 16/30 Read block 360 len 4096 17/30 Read block 520 len 24576 18/30 Read block 297 len 8192 19/30 Write block 345 len 4096 data: 5 20/30 Read block 552 len 16384 21/30 Read block 354 len 4096 22/30 Write block 542 len 28672 data: 4 5 6 6 6 6 7 23/30 Write block 341 len 4096 data: 6 24/30 Read block 458 len 36864 25/30 Read block 36 len 16384 26/30 Write block 114 len 4096 data: 3 27/30 Write block 118 len 32768 data: 6 8 8 7 7 8 7 7 28/30 Write block 213 len 40960 data: 3 4 3 3 3 3 3 3 5 5 29/30 Write block 536 len 32768 data: 4 4 4 2 2 3 5 6 30/30 Read block 365 len 36864 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Read 9 Done Done Done false 1003 Acked Read 8 Done Done Done false 1004 Acked Read 10 Done Done Done false 1005 Acked Read 4 Done Done Done false 1006 Acked Read 9 Done Done Done false 1007 Acked Write 9 Done Done Done false 1008 Acked Write 10 Done Done Done false 1009 Acked Read 9 Done Done Done false 1010 Acked Read 5 Done Done Done false 1011 Acked Read 10 Done Done Done false 1012 Acked Read 8 Done Done Done false 1013 Acked Write 10 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 8 Sent Done Done false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 28 30 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:32:07.730185227Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1339} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 31 31 31 F F F 1 020-039 35 35 35 49 49 49 F F F 2 040-059 37 37 37 51 51 51 F F F 3 060-079 34 34 34 48 48 48 F F F 4 080-099 34 34 34 48 48 48 T T T 5 100-119 37 37 37 51 51 51 T T T 6 120-139 37 37 37 51 51 51 T T T 7 140-159 33 33 33 47 47 47 F F F 8 160-179 37 37 37 51 51 51 T T T 9 180-199 37 37 37 51 51 51 T T T 10 200-219 37 37 37 51 51 51 T T T 11 220-239 37 37 37 51 51 51 T T T 12 240-259 37 37 37 51 51 51 F F F 13 260-279 37 37 37 51 51 51 F F F 14 280-299 28 28 28 42 42 42 F F F 15 300-319 32 32 32 44 44 44 F F F 16 320-339 35 35 35 49 49 49 F F F 17 340-359 37 37 37 51 51 51 T T T 18 360-379 37 37 37 51 51 51 F F F 19 380-399 37 37 37 51 51 51 F F F 20 400-419 37 37 37 51 51 51 F F F 21 420-439 33 33 33 47 47 47 T T T 22 440-459 36 36 36 50 50 50 T T T 23 460-479 28 28 28 42 42 42 F F F 24 480-499 37 37 37 51 51 51 F F F 25 500-519 35 35 35 49 49 49 F F F 26 520-539 35 35 35 49 49 49 T T T 27 540-559 37 37 37 51 51 51 T T T 28 560-579 32 32 32 44 44 44 F F F 29 580-599 37 37 37 51 51 51 F F F Max gen: 37, Max flush: 51 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 31 31 31 F F F 1 020-039 35 35 35 49 49 49 F F F 2 040-059 37 37 37 51 51 51 F F F 3 060-079 34 34 34 48 48 48 F F F 4 080-099 38 38 38 52 52 52 F F F 5 100-119 38 38 38 52 52 52 F F F 6 120-139 38 38 38 52 52 52 F F F 7 140-159 33 33 33 47 47 47 F F F 8 160-179 38 38 38 52 52 52 F F F 9 180-199 38 38 38 52 52 52 F F F 10 200-219 38 38 38 52 52 52 F F F 11 220-239 38 38 38 52 52 52 F F F 12 240-259 37 37 37 51 51 51 F F F 13 260-279 37 37 37 51 51 51 F F F 14 280-299 28 28 28 42 42 42 F F F 15 300-319 32 32 32 44 44 44 F F F 16 320-339 35 35 35 49 49 49 F F F 17 340-359 38 38 38 52 52 52 F F F 18 360-379 37 37 37 51 51 51 F F F 19 380-399 37 37 37 51 51 51 F F F 20 400-419 37 37 37 51 51 51 F F F 21 420-439 38 38 38 52 52 52 F F F 22 440-459 38 38 38 52 52 52 F F F 23 460-479 28 28 28 42 42 42 F F F 24 480-499 37 37 37 51 51 51 F F F 25 500-519 35 35 35 49 49 49 F F F 26 520-539 38 38 38 52 52 52 F F F 27 540-559 38 38 38 52 52 52 F F F 28 560-579 32 32 32 44 44 44 F F F 29 580-599 37 37 37 51 51 51 F F F Max gen: 38, Max flush: 52 Begin loop 20 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.03509632Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:32:12.076846874Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.07721296Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.108546555Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.109180193Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.109213818Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.109231515Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.109246403Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347} {"msg":"Crucible e9654a61-6ec1-41fe-b662-73e4fe09026f has session id: 20e51b9f-e7ab-419d-94d4-359cda4db343","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.1096792Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"Upstairs opts: Upstairs UUID: e9654a61-6ec1-41fe-b662-73e4fe09026f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.109723963Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"Crucible stats registered with UUID: e9654a61-6ec1-41fe-b662-73e4fe09026f","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.1097459Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.139870731Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.139900777Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347} {"msg":"e9654a61-6ec1-41fe-b662-73e4fe09026f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.139934032Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.139951089Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"client":"0","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.139977406Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"client":"0","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.139995183Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"client":"1","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.14001328Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"client":"1","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.140030438Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"client":"2","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.140047925Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"client":"2","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.140074411Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"":"io task","client":"2","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.140112966Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"":"io task","client":"0","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.140147651Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"":"io task","client":"1","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"ds_connection connected from Ok(127.0.0.1:34707)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.140180676Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"":"io task","client":"2","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"ds_connection connected from Ok(127.0.0.1:41579)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.140200003Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"":"io task","client":"1","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"ds_connection connected from Ok(127.0.0.1:45581)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.140441228Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"":"io task","client":"0","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.141014844Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"client":"1","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.14104453Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"client":"1","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.141061657Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"client":"2","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.141078215Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"client":"2","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.141105571Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"client":"0","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.141133377Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"client":"0","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.141596979Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"client":"1","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.141679807Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"client":"0","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.141783912Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"client":"2","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.142173055Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"client":"1","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.142199431Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"client":"1","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.142214929Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.142296777Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"client":"0","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.142331782Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"client":"0","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.142354139Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.142367527Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"client":"2","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.142381515Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"client":"2","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"[0]R flush_numbers[0..12]: [31, 49, 51, 48, 52, 52, 52, 47, 52, 52, 52, 52]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.142395503Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"":"downstairs","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"[0]R generation[0..12]: [21, 35, 37, 34, 38, 38, 38, 33, 38, 38, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.142409181Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"":"downstairs","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.142423099Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"":"downstairs","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"[1]R flush_numbers[0..12]: [31, 49, 51, 48, 52, 52, 52, 47, 52, 52, 52, 52]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.142436957Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"":"downstairs","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"[1]R generation[0..12]: [21, 35, 37, 34, 38, 38, 38, 33, 38, 38, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.142451804Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"":"downstairs","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.142465812Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"":"downstairs","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"[2]R flush_numbers[0..12]: [31, 49, 51, 48, 52, 52, 52, 47, 52, 52, 52, 52]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.1424795Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"":"downstairs","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"[2]R generation[0..12]: [21, 35, 37, 34, 38, 38, 38, 33, 38, 38, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.142493588Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"":"downstairs","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.142514995Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"":"downstairs","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"Max found gen is 39","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:32:12.142540371Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"":"downstairs","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"Generation requested: 40 >= found:39","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.142571497Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"":"downstairs","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"Next flush: 53","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.142599063Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"":"downstairs","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.142626739Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"":"downstairs","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.142655665Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.142683181Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"client":"0","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.142711026Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"client":"1","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.142742382Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"client":"2","":"downstairs","session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.142771908Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.142787465Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"e9654a61-6ec1-41fe-b662-73e4fe09026f is now active with session: 20e51b9f-e7ab-419d-94d4-359cda4db343","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.142800604Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.142812992Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347,"session_id":"20e51b9f-e7ab-419d-94d4-359cda4db343"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:12.14282546Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 566 len 4096 data: 3 02/30 Write block 411 len 24576 data: 7 8 9 11 10 9 03/30 Write block 564 len 4096 data: 4 04/30 Write block 149 len 16384 data: 4 4 4 3 05/30 Read block 455 len 8192 06/30 Write block 308 len 32768 data: 4 4 4 4 4 4 4 4 07/30 Read block 282 len 8192 08/30 Write block 87 len 28672 data: 3 3 4 4 3 3 3 09/30 Write block 333 len 32768 data: 3 4 3 3 3 4 6 6 10/30 Flush 11/30 Write block 349 len 24576 data: 3 3 4 5 3 3 12/30 Read block 121 len 32768 13/30 Write block 145 len 4096 data: 5 14/30 Read block 354 len 24576 15/30 Read block 73 len 32768 16/30 Read block 230 len 8192 17/30 Write block 387 len 36864 data: 3 3 3 3 5 6 6 6 5 18/30 Write block 475 len 4096 data: 5 19/30 Write block 476 len 8192 data: 5 4 20/30 Flush 21/30 Write block 42 len 20480 data: 5 4 4 5 6 22/30 Write block 88 len 28672 data: 4 5 5 4 4 4 3 23/30 Read block 183 len 36864 24/30 Write block 153 len 40960 data: 2 2 2 2 2 4 5 7 7 8 25/30 Read block 469 len 36864 26/30 Read block 422 len 28672 27/30 Read block 465 len 40960 28/30 Read block 199 len 12288 29/30 Read block 230 len 32768 30/30 Read block 525 len 28672 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 5 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 7 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 10 9 10 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:32:12.161917022Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1347} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 31 31 31 F F F 1 020-039 35 35 35 49 49 49 F F F 2 040-059 37 37 37 51 51 51 T T T 3 060-079 34 34 34 48 48 48 F F F 4 080-099 40 40 40 53 53 53 T T T 5 100-119 38 38 38 52 52 52 F F F 6 120-139 38 38 38 52 52 52 F F F 7 140-159 40 40 40 54 54 54 T T T 8 160-179 38 38 38 52 52 52 T T T 9 180-199 38 38 38 52 52 52 F F F 10 200-219 38 38 38 52 52 52 F F F 11 220-239 38 38 38 52 52 52 F F F 12 240-259 37 37 37 51 51 51 F F F 13 260-279 37 37 37 51 51 51 F F F 14 280-299 28 28 28 42 42 42 F F F 15 300-319 40 40 40 53 53 53 F F F 16 320-339 40 40 40 53 53 53 F F F 17 340-359 40 40 40 54 54 54 F F F 18 360-379 37 37 37 51 51 51 F F F 19 380-399 40 40 40 54 54 54 F F F 20 400-419 40 40 40 53 53 53 F F F 21 420-439 38 38 38 52 52 52 F F F 22 440-459 38 38 38 52 52 52 F F F 23 460-479 40 40 40 54 54 54 F F F 24 480-499 37 37 37 51 51 51 F F F 25 500-519 35 35 35 49 49 49 F F F 26 520-539 38 38 38 52 52 52 F F F 27 540-559 38 38 38 52 52 52 F F F 28 560-579 40 40 40 53 53 53 F F F 29 580-599 37 37 37 51 51 51 F F F Max gen: 40, Max flush: 54 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 31 31 31 F F F 1 020-039 35 35 35 49 49 49 F F F 2 040-059 41 41 41 55 55 55 F F F 3 060-079 34 34 34 48 48 48 F F F 4 080-099 41 41 41 55 55 55 F F F 5 100-119 38 38 38 52 52 52 F F F 6 120-139 38 38 38 52 52 52 F F F 7 140-159 41 41 41 55 55 55 F F F 8 160-179 41 41 41 55 55 55 F F F 9 180-199 38 38 38 52 52 52 F F F 10 200-219 38 38 38 52 52 52 F F F 11 220-239 38 38 38 52 52 52 F F F 12 240-259 37 37 37 51 51 51 F F F 13 260-279 37 37 37 51 51 51 F F F 14 280-299 28 28 28 42 42 42 F F F 15 300-319 40 40 40 53 53 53 F F F 16 320-339 40 40 40 53 53 53 F F F 17 340-359 40 40 40 54 54 54 F F F 18 360-379 37 37 37 51 51 51 F F F 19 380-399 40 40 40 54 54 54 F F F 20 400-419 40 40 40 53 53 53 F F F 21 420-439 38 38 38 52 52 52 F F F 22 440-459 38 38 38 52 52 52 F F F 23 460-479 40 40 40 54 54 54 F F F 24 480-499 37 37 37 51 51 51 F F F 25 500-519 35 35 35 49 49 49 F F F 26 520-539 38 38 38 52 52 52 F F F 27 540-559 38 38 38 52 52 52 F F F 28 560-579 40 40 40 53 53 53 F F F 29 580-599 37 37 37 51 51 51 F F F Max gen: 41, Max flush: 55 Begin loop 21 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.451689126Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.495455435Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.496086413Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.496131037Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.496148994Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.496167611Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355} {"msg":"Crucible e09f575b-c88c-4615-b8a7-32abddf8c46b has session id: 3d5dd069-7c83-46ed-b847-5714f00193ca","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.49665304Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"Upstairs opts: Upstairs UUID: e09f575b-c88c-4615-b8a7-32abddf8c46b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.496693744Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"Crucible stats registered with UUID: e09f575b-c88c-4615-b8a7-32abddf8c46b","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.496710062Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.527302995Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.52733569Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355} {"msg":"e09f575b-c88c-4615-b8a7-32abddf8c46b active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.527370295Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.527388293Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"client":"0","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.527416818Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"client":"0","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.527435426Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"client":"1","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.527453503Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"client":"1","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.52747115Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"client":"2","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.527536491Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"client":"2","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.527580484Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"":"io task","client":"2","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.52761216Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"":"io task","client":"0","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.527641216Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"":"io task","client":"1","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:55497)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.527671311Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"":"io task","client":"2","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:54706)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.527802152Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"":"io task","client":"1","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:34159)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.52788281Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"":"io task","client":"0","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.528351882Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"client":"1","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.528392216Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"client":"1","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.528422231Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"client":"0","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.528496911Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"client":"0","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.528539014Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"client":"2","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.52857308Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"client":"2","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.529007236Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"client":"1","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.529055599Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"client":"0","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.529104632Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"client":"2","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.529606798Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"client":"1","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.529647592Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"client":"1","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.529678518Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.529702504Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"client":"0","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.529726571Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"client":"0","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.529754207Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.529777864Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"client":"2","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.5298025Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"client":"2","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"[0]R flush_numbers[0..12]: [31, 49, 55, 48, 55, 52, 52, 55, 55, 52, 52, 52]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.529826646Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"":"downstairs","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"[0]R generation[0..12]: [21, 35, 41, 34, 41, 38, 38, 41, 41, 38, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.529851593Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"":"downstairs","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.529879329Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"":"downstairs","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"[1]R flush_numbers[0..12]: [31, 49, 55, 48, 55, 52, 52, 55, 55, 52, 52, 52]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.529906265Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"":"downstairs","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"[1]R generation[0..12]: [21, 35, 41, 34, 41, 38, 38, 41, 41, 38, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.529932381Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"":"downstairs","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.529961367Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"[2]R flush_numbers[0..12]: [31, 49, 55, 48, 55, 52, 52, 55, 55, 52, 52, 52]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.529998961Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"":"downstairs","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"[2]R generation[0..12]: [21, 35, 41, 34, 41, 38, 38, 41, 41, 38, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.530031136Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"":"downstairs","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.530062552Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"":"downstairs","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.530091508Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"":"downstairs","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.530119934Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"":"downstairs","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"Next flush: 56","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.530151159Z"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"":"downstairs","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.530184144Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"":"downstairs","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.530209501Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.530222919Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"client":"0","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.530236417Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"client":"1","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.530250804Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"client":"2","":"downstairs","session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.530264193Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.530276651Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"e09f575b-c88c-4615-b8a7-32abddf8c46b is now active with session: 3d5dd069-7c83-46ed-b847-5714f00193ca","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.530289619Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.530301967Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355,"session_id":"3d5dd069-7c83-46ed-b847-5714f00193ca"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:16.530314415Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 294 len 36864 data: 4 3 4 5 4 4 4 4 4 02/30 Read block 97 len 4096 03/30 Write block 592 len 20480 data: 5 5 5 4 3 04/30 Write block 25 len 40960 data: 3 3 3 3 4 3 2 2 3 3 05/30 Read block 343 len 24576 06/30 Read block 440 len 16384 07/30 Read block 40 len 40960 08/30 Read block 541 len 28672 09/30 Read block 250 len 12288 10/30 Flush 11/30 Write block 389 len 28672 data: 4 4 6 7 7 7 6 12/30 Read block 527 len 20480 13/30 Read block 86 len 40960 14/30 Read block 203 len 24576 15/30 Write block 30 len 16384 data: 4 3 3 4 16/30 Read block 280 len 40960 17/30 Read block 180 len 36864 18/30 Read block 361 len 32768 19/30 Write block 500 len 12288 data: 5 4 5 20/30 Write block 112 len 40960 data: 2 3 4 4 4 4 7 9 9 8 21/30 Flush 22/30 Write block 584 len 24576 data: 7 5 5 5 6 5 23/30 Write block 7 len 8192 data: 2 2 24/30 Read block 116 len 12288 25/30 Read block 507 len 8192 26/30 Read block 36 len 24576 27/30 Write block 479 len 20480 data: 4 4 3 3 3 28/30 Read block 469 len 8192 29/30 Write block 563 len 8192 data: 4 5 30/30 Write block 414 len 20480 data: 12 11 10 9 7 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 6 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:32:16.550838488Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1355} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 31 31 31 T T T 1 020-039 42 42 42 57 57 57 F F F 2 040-059 41 41 41 55 55 55 F F F 3 060-079 34 34 34 48 48 48 F F F 4 080-099 41 41 41 55 55 55 F F F 5 100-119 42 42 42 57 57 57 F F F 6 120-139 42 42 42 57 57 57 F F F 7 140-159 41 41 41 55 55 55 F F F 8 160-179 41 41 41 55 55 55 F F F 9 180-199 38 38 38 52 52 52 F F F 10 200-219 38 38 38 52 52 52 F F F 11 220-239 38 38 38 52 52 52 F F F 12 240-259 37 37 37 51 51 51 F F F 13 260-279 37 37 37 51 51 51 F F F 14 280-299 42 42 42 56 56 56 F F F 15 300-319 42 42 42 56 56 56 F F F 16 320-339 40 40 40 53 53 53 F F F 17 340-359 40 40 40 54 54 54 F F F 18 360-379 37 37 37 51 51 51 F F F 19 380-399 42 42 42 57 57 57 F F F 20 400-419 40 40 40 53 53 53 T T T 21 420-439 38 38 38 52 52 52 F F F 22 440-459 38 38 38 52 52 52 F F F 23 460-479 40 40 40 54 54 54 T T T 24 480-499 37 37 37 51 51 51 T T T 25 500-519 42 42 42 57 57 57 F F F 26 520-539 38 38 38 52 52 52 F F F 27 540-559 38 38 38 52 52 52 F F F 28 560-579 40 40 40 53 53 53 T T T 29 580-599 42 42 42 56 56 56 T T T Max gen: 42, Max flush: 57 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 58 58 58 F F F 1 020-039 42 42 42 57 57 57 F F F 2 040-059 41 41 41 55 55 55 F F F 3 060-079 34 34 34 48 48 48 F F F 4 080-099 41 41 41 55 55 55 F F F 5 100-119 42 42 42 57 57 57 F F F 6 120-139 42 42 42 57 57 57 F F F 7 140-159 41 41 41 55 55 55 F F F 8 160-179 41 41 41 55 55 55 F F F 9 180-199 38 38 38 52 52 52 F F F 10 200-219 38 38 38 52 52 52 F F F 11 220-239 38 38 38 52 52 52 F F F 12 240-259 37 37 37 51 51 51 F F F 13 260-279 37 37 37 51 51 51 F F F 14 280-299 42 42 42 56 56 56 F F F 15 300-319 42 42 42 56 56 56 F F F 16 320-339 40 40 40 53 53 53 F F F 17 340-359 40 40 40 54 54 54 F F F 18 360-379 37 37 37 51 51 51 F F F 19 380-399 42 42 42 57 57 57 F F F 20 400-419 43 43 43 58 58 58 F F F 21 420-439 38 38 38 52 52 52 F F F 22 440-459 38 38 38 52 52 52 F F F 23 460-479 43 43 43 58 58 58 F F F 24 480-499 43 43 43 58 58 58 F F F 25 500-519 42 42 42 57 57 57 F F F 26 520-539 38 38 38 52 52 52 F F F 27 540-559 38 38 38 52 52 52 F F F 28 560-579 43 43 43 58 58 58 F F F 29 580-599 43 43 43 58 58 58 F F F Max gen: 43, Max flush: 58 Begin loop 22 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.819636574Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.863346351Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.864028792Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.864064846Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.864079854Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.864094212Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363} {"msg":"Crucible 85907fe8-8d07-4787-8cf0-61a9e31c1d57 has session id: b4f9d418-7ee9-4f74-aa9e-992766f334f7","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.864501973Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"Upstairs opts: Upstairs UUID: 85907fe8-8d07-4787-8cf0-61a9e31c1d57, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.864538157Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"Crucible stats registered with UUID: 85907fe8-8d07-4787-8cf0-61a9e31c1d57","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.864554545Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.895362687Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.895400951Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363} {"msg":"85907fe8-8d07-4787-8cf0-61a9e31c1d57 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.895436056Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.895457603Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"client":"0","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.895502426Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"client":"0","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.895573506Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"client":"1","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.89561484Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"client":"1","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.895647245Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"client":"2","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.89568082Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"client":"2","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.895700997Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"":"io task","client":"2","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.895719415Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"":"io task","client":"0","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.895740192Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"":"io task","client":"1","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:40654)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.895772857Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"":"io task","client":"2","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:33136)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.895808031Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"":"io task","client":"1","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:42305)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.895836997Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"":"io task","client":"0","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.896392886Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"client":"0","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.89643508Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"client":"0","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.896532956Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"client":"1","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.896628432Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"client":"1","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.896748614Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"client":"2","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.8967794Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"client":"2","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.897023534Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"client":"1","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.897163924Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"client":"0","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.897500544Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"client":"2","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.897646923Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"client":"1","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.897682618Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"client":"1","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.897698775Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.897711744Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"client":"0","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.897725571Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"client":"0","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.89773877Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.897952518Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"client":"2","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.897972475Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"client":"2","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"[0]R flush_numbers[0..12]: [58, 57, 55, 48, 55, 57, 57, 55, 55, 52, 52, 52]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.897986873Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"":"downstairs","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"[0]R generation[0..12]: [43, 42, 41, 34, 41, 42, 42, 41, 41, 38, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.898004941Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"":"downstairs","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.898019679Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"":"downstairs","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"[1]R flush_numbers[0..12]: [58, 57, 55, 48, 55, 57, 57, 55, 55, 52, 52, 52]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.898033407Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"":"downstairs","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"[1]R generation[0..12]: [43, 42, 41, 34, 41, 42, 42, 41, 41, 38, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.898047185Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"":"downstairs","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.898061142Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"":"downstairs","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"[2]R flush_numbers[0..12]: [58, 57, 55, 48, 55, 57, 57, 55, 55, 52, 52, 52]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.89807478Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"":"downstairs","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"[2]R generation[0..12]: [43, 42, 41, 34, 41, 42, 42, 41, 41, 38, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.898088319Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"":"downstairs","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.898102107Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"":"downstairs","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.898115525Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"":"downstairs","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.898128893Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"":"downstairs","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"Next flush: 59","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.898142091Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"":"downstairs","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.898155199Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"":"downstairs","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.898168387Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:32:20.898180545Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"client":"0","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.898205912Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"client":"1","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.89821961Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"client":"2","":"downstairs","session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.898233157Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.898245546Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"85907fe8-8d07-4787-8cf0-61a9e31c1d57 is now active with session: b4f9d418-7ee9-4f74-aa9e-992766f334f7","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.898258544Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.898270672Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363,"session_id":"b4f9d418-7ee9-4f74-aa9e-992766f334f7"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:20.89828309Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 18 len 28672 data: 4 5 4 5 5 5 5 02/30 Flush 03/30 Write block 378 len 8192 data: 4 4 04/30 Read block 18 len 20480 05/30 Flush 06/30 Write block 256 len 32768 data: 7 7 7 7 6 4 4 4 07/30 Read block 511 len 12288 08/30 Write block 154 len 12288 data: 3 3 3 09/30 Read block 477 len 8192 10/30 Flush 11/30 Write block 283 len 28672 data: 4 4 4 3 3 3 3 12/30 Read block 99 len 24576 13/30 Write block 270 len 24576 data: 3 4 2 2 2 3 14/30 Read block 433 len 40960 15/30 Read block 215 len 16384 16/30 Read block 434 len 16384 17/30 Write block 455 len 24576 data: 6 4 4 5 6 5 18/30 Write block 493 len 36864 data: 5 4 5 5 5 5 5 6 5 19/30 Read block 465 len 12288 20/30 Write block 464 len 40960 data: 4 4 2 3 2 4 4 4 4 4 21/30 Read block 577 len 36864 22/30 Read block 14 len 8192 23/30 Read block 202 len 4096 24/30 Write block 341 len 32768 data: 7 6 4 5 6 5 4 3 25/30 Read block 119 len 4096 26/30 Read block 464 len 32768 27/30 Read block 264 len 4096 28/30 Write block 351 len 4096 data: 5 29/30 Read block 249 len 28672 30/30 Read block 276 len 16384 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 7 Done Done Done false 1011 Acked Read 6 Done Done Done false 1012 Acked Write 6 Done Done Done false 1013 Acked Read 10 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 20 19 19 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1009 1009 1009 Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:32:20.918607262Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1363} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 59 59 59 F F F 1 020-039 44 44 44 59 59 59 F F F 2 040-059 41 41 41 55 55 55 F F F 3 060-079 34 34 34 48 48 48 F F F 4 080-099 41 41 41 55 55 55 F F F 5 100-119 42 42 42 57 57 57 F F F 6 120-139 42 42 42 57 57 57 F F F 7 140-159 44 44 44 61 61 61 F F F 8 160-179 41 41 41 55 55 55 F F F 9 180-199 38 38 38 52 52 52 F F F 10 200-219 38 38 38 52 52 52 F F F 11 220-239 38 38 38 52 52 52 F F F 12 240-259 44 44 44 61 61 61 F F F 13 260-279 44 44 44 61 61 61 T T T 14 280-299 42 42 42 56 56 56 T T T 15 300-319 42 42 42 56 56 56 F F F 16 320-339 40 40 40 53 53 53 F F F 17 340-359 40 40 40 54 54 54 T T T 18 360-379 44 44 44 60 60 60 F F F 19 380-399 42 42 42 57 57 57 F F F 20 400-419 43 43 43 58 58 58 F F F 21 420-439 38 38 38 52 52 52 F F F 22 440-459 38 38 38 52 52 52 T T T 23 460-479 43 43 43 58 58 58 T T T 24 480-499 43 43 43 58 58 58 T T T 25 500-519 42 42 42 57 57 57 T T T 26 520-539 38 38 38 52 52 52 F F F 27 540-559 38 38 38 52 52 52 F F F 28 560-579 43 43 43 58 58 58 F F F 29 580-599 43 43 43 58 58 58 F F F Max gen: 44, Max flush: 61 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 59 59 59 F F F 1 020-039 44 44 44 59 59 59 F F F 2 040-059 41 41 41 55 55 55 F F F 3 060-079 34 34 34 48 48 48 F F F 4 080-099 41 41 41 55 55 55 F F F 5 100-119 42 42 42 57 57 57 F F F 6 120-139 42 42 42 57 57 57 F F F 7 140-159 44 44 44 61 61 61 F F F 8 160-179 41 41 41 55 55 55 F F F 9 180-199 38 38 38 52 52 52 F F F 10 200-219 38 38 38 52 52 52 F F F 11 220-239 38 38 38 52 52 52 F F F 12 240-259 44 44 44 61 61 61 F F F 13 260-279 45 45 45 62 62 62 F F F 14 280-299 45 45 45 62 62 62 F F F 15 300-319 42 42 42 56 56 56 F F F 16 320-339 40 40 40 53 53 53 F F F 17 340-359 45 45 45 62 62 62 F F F 18 360-379 44 44 44 60 60 60 F F F 19 380-399 42 42 42 57 57 57 F F F 20 400-419 43 43 43 58 58 58 F F F 21 420-439 38 38 38 52 52 52 F F F 22 440-459 45 45 45 62 62 62 F F F 23 460-479 45 45 45 62 62 62 F F F 24 480-499 45 45 45 62 62 62 F F F 25 500-519 45 45 45 62 62 62 F F F 26 520-539 38 38 38 52 52 52 F F F 27 540-559 38 38 38 52 52 52 F F F 28 560-579 43 43 43 58 58 58 F F F 29 580-599 43 43 43 58 58 58 F F F Max gen: 45, Max flush: 62 Begin loop 23 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.192327969Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.235697676Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.236442668Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.236486321Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.236504119Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.236522596Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371} {"msg":"Crucible 584a1968-19ab-44f0-810f-4fdecd4ce99c has session id: 2c1ba5b3-d118-4670-ad42-c450da4324d9","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.236928406Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"Upstairs opts: Upstairs UUID: 584a1968-19ab-44f0-810f-4fdecd4ce99c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.236987078Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"Crucible stats registered with UUID: 584a1968-19ab-44f0-810f-4fdecd4ce99c","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.237018843Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.267231602Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.267263707Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371} {"msg":"584a1968-19ab-44f0-810f-4fdecd4ce99c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.267293703Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.26731065Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"client":"0","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.267342576Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"client":"0","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.267361003Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"client":"1","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.26737936Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"client":"1","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.267410406Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"client":"2","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.267443681Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"client":"2","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.267474886Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"":"io task","client":"2","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.267510381Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"":"io task","client":"0","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.267546296Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"":"io task","client":"1","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:36080)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.267582381Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"":"io task","client":"0","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:43788)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.267866019Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"":"io task","client":"2","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:35861)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.267909803Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"":"io task","client":"1","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.268175354Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"client":"0","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.268227956Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"client":"0","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.268349059Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"client":"2","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.268374355Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"client":"2","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.268568587Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"client":"1","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.268601602Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"client":"1","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.268990745Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"client":"2","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.269021191Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"client":"0","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.269044027Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"client":"1","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.269529646Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"client":"2","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.269558432Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"client":"2","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.26957321Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.269651319Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"client":"1","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.269670926Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"client":"1","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.269684814Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.269697012Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"client":"0","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.26971064Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"client":"0","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"[0]R flush_numbers[0..12]: [59, 59, 55, 48, 55, 57, 57, 61, 55, 52, 52, 52]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.269724538Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"":"downstairs","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"[0]R generation[0..12]: [44, 44, 41, 34, 41, 42, 42, 44, 41, 38, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.269738686Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"":"downstairs","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.269753034Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"":"downstairs","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"[1]R flush_numbers[0..12]: [59, 59, 55, 48, 55, 57, 57, 61, 55, 52, 52, 52]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.269770301Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"":"downstairs","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"[1]R generation[0..12]: [44, 44, 41, 34, 41, 42, 42, 44, 41, 38, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.269784459Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"":"downstairs","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.269798547Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"":"downstairs","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"[2]R flush_numbers[0..12]: [59, 59, 55, 48, 55, 57, 57, 61, 55, 52, 52, 52]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.269812335Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"":"downstairs","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"[2]R generation[0..12]: [44, 44, 41, 34, 41, 42, 42, 44, 41, 38, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.269826233Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"":"downstairs","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.269840411Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"":"downstairs","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.269853579Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"":"downstairs","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.269866977Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"":"downstairs","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"Next flush: 63","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.269879935Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.269901112Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"":"downstairs","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.26991465Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.269926948Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"client":"0","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.269940256Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"client":"1","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.269953524Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"client":"2","":"downstairs","session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.269969132Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.26998185Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"584a1968-19ab-44f0-810f-4fdecd4ce99c is now active with session: 2c1ba5b3-d118-4670-ad42-c450da4324d9","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.269995138Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.270007307Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371,"session_id":"2c1ba5b3-d118-4670-ad42-c450da4324d9"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:25.270019575Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 89 len 24576 data: 6 6 5 5 5 4 02/30 Read block 11 len 40960 03/30 Read block 581 len 12288 04/30 Write block 63 len 36864 data: 5 7 5 5 6 5 3 5 5 05/30 Write block 396 len 20480 data: 7 7 7 7 7 06/30 Read block 117 len 4096 07/30 Read block 129 len 4096 08/30 Write block 8 len 8192 data: 3 2 09/30 Read block 552 len 40960 10/30 Write block 76 len 32768 data: 6 5 5 5 5 5 4 4 11/30 Write block 250 len 20480 data: 6 5 8 7 7 12/30 Read block 505 len 24576 13/30 Flush 14/30 Write block 542 len 4096 data: 6 15/30 Read block 442 len 32768 16/30 Write block 80 len 8192 data: 6 6 17/30 Flush 18/30 Flush 19/30 Write block 235 len 36864 data: 7 7 5 5 7 7 8 8 7 20/30 Flush 21/30 Write block 578 len 4096 data: 5 22/30 Read block 514 len 8192 23/30 Write block 13 len 36864 data: 3 4 3 3 4 5 6 5 6 24/30 Write block 517 len 16384 data: 4 4 3 4 25/30 Write block 562 len 4096 data: 4 26/30 Write block 391 len 20480 data: 7 8 8 8 7 27/30 Read block 45 len 16384 28/30 Write block 389 len 16384 data: 5 5 8 9 29/30 Read block 301 len 16384 30/30 Read block 209 len 20480 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 1 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 5 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 10 10 9 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:32:25.288180043Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1371} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 63 63 63 T T T 1 020-039 44 44 44 59 59 59 T T T 2 040-059 41 41 41 55 55 55 F F F 3 060-079 46 46 46 63 63 63 F F F 4 080-099 46 46 46 64 64 64 F F F 5 100-119 42 42 42 57 57 57 F F F 6 120-139 42 42 42 57 57 57 F F F 7 140-159 44 44 44 61 61 61 F F F 8 160-179 41 41 41 55 55 55 F F F 9 180-199 38 38 38 52 52 52 F F F 10 200-219 38 38 38 52 52 52 F F F 11 220-239 46 46 46 66 66 66 F F F 12 240-259 46 46 46 66 66 66 F F F 13 260-279 45 45 45 62 62 62 F F F 14 280-299 45 45 45 62 62 62 F F F 15 300-319 42 42 42 56 56 56 F F F 16 320-339 40 40 40 53 53 53 F F F 17 340-359 45 45 45 62 62 62 F F F 18 360-379 44 44 44 60 60 60 F F F 19 380-399 46 46 46 63 63 63 T T T 20 400-419 46 46 46 63 63 63 F F F 21 420-439 38 38 38 52 52 52 F F F 22 440-459 45 45 45 62 62 62 F F F 23 460-479 45 45 45 62 62 62 F F F 24 480-499 45 45 45 62 62 62 F F F 25 500-519 45 45 45 62 62 62 T T T 26 520-539 38 38 38 52 52 52 T T T 27 540-559 46 46 46 64 64 64 F F F 28 560-579 43 43 43 58 58 58 T T T 29 580-599 43 43 43 58 58 58 F F F Max gen: 46, Max flush: 66 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 67 67 67 F F F 1 020-039 47 47 47 67 67 67 F F F 2 040-059 41 41 41 55 55 55 F F F 3 060-079 46 46 46 63 63 63 F F F 4 080-099 46 46 46 64 64 64 F F F 5 100-119 42 42 42 57 57 57 F F F 6 120-139 42 42 42 57 57 57 F F F 7 140-159 44 44 44 61 61 61 F F F 8 160-179 41 41 41 55 55 55 F F F 9 180-199 38 38 38 52 52 52 F F F 10 200-219 38 38 38 52 52 52 F F F 11 220-239 46 46 46 66 66 66 F F F 12 240-259 46 46 46 66 66 66 F F F 13 260-279 45 45 45 62 62 62 F F F 14 280-299 45 45 45 62 62 62 F F F 15 300-319 42 42 42 56 56 56 F F F 16 320-339 40 40 40 53 53 53 F F F 17 340-359 45 45 45 62 62 62 F F F 18 360-379 44 44 44 60 60 60 F F F 19 380-399 47 47 47 67 67 67 F F F 20 400-419 46 46 46 63 63 63 F F F 21 420-439 38 38 38 52 52 52 F F F 22 440-459 45 45 45 62 62 62 F F F 23 460-479 45 45 45 62 62 62 F F F 24 480-499 45 45 45 62 62 62 F F F 25 500-519 47 47 47 67 67 67 F F F 26 520-539 47 47 47 67 67 67 F F F 27 540-559 46 46 46 64 64 64 F F F 28 560-579 47 47 47 67 67 67 F F F 29 580-599 43 43 43 58 58 58 F F F Max gen: 47, Max flush: 67 Begin loop 24 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.561190523Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.605487215Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.606128591Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.606170275Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.606185913Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.606200811Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379} {"msg":"Crucible 033bd1aa-cea2-433a-ba8f-ec87e384ac75 has session id: 70b25287-1744-4c51-92cb-0132403b057f","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.606669193Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"Upstairs opts: Upstairs UUID: 033bd1aa-cea2-433a-ba8f-ec87e384ac75, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.606703807Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"Crucible stats registered with UUID: 033bd1aa-cea2-433a-ba8f-ec87e384ac75","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.606719485Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.637626182Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.637695552Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379} {"msg":"033bd1aa-cea2-433a-ba8f-ec87e384ac75 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.637722858Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.637747345Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"client":"0","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.637782119Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"client":"0","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.637815785Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"client":"1","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.637870177Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"client":"1","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.637906351Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"client":"2","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.637956454Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"client":"2","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.63818833Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"":"io task","client":"2","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.638240113Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"":"io task","client":"0","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.638269808Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"":"io task","client":"1","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"ds_connection connected from Ok(127.0.0.1:33454)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.638299604Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"":"io task","client":"1","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50788)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.638316201Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"":"io task","client":"0","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"ds_connection connected from Ok(127.0.0.1:56834)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.638338288Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"":"io task","client":"2","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.638947069Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"client":"2","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.639003591Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"client":"2","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.639045795Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"client":"0","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.639087529Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"client":"0","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.639136672Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"client":"1","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.639176246Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"client":"1","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.63976512Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"client":"0","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.639815113Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"client":"2","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.639873184Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"client":"1","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.640606647Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"client":"2","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.640673148Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"client":"2","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.64072027Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.640761535Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"client":"1","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.640803249Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"client":"1","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.640843772Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.640876398Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"client":"0","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.640916832Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"client":"0","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"[0]R flush_numbers[0..12]: [67, 67, 55, 63, 64, 57, 57, 61, 55, 52, 52, 66]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.640956266Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"":"downstairs","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"[0]R generation[0..12]: [47, 47, 41, 46, 46, 42, 42, 44, 41, 38, 38, 46]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.64099746Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"":"downstairs","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.641034795Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"":"downstairs","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"[1]R flush_numbers[0..12]: [67, 67, 55, 63, 64, 57, 57, 61, 55, 52, 52, 66]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.641069479Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"":"downstairs","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"[1]R generation[0..12]: [47, 47, 41, 46, 46, 42, 42, 44, 41, 38, 38, 46]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.641114753Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"":"downstairs","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"}Wait for a query_work_queue command to finish before sending IO {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.641167125Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"":"downstairs","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"[2]R flush_numbers[0..12]: [67, 67, 55, 63, 64, 57, 57, 61, 55, 52, 52, 66]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.641219997Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"":"downstairs","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"[2]R generation[0..12]: [47, 47, 41, 46, 46, 42, 42, 44, 41, 38, 38, 46]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.64126859Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"":"downstairs","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.641315644Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"":"downstairs","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.641354038Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"":"downstairs","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.641398891Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"":"downstairs","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"}Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Next flush: 68","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.641446644Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"":"downstairs","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.641489308Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"":"downstairs","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.641515954Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.641535851Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"client":"0","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.641556319Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"client":"1","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.641576205Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"client":"2","":"downstairs","session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.641596542Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.64161613Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"033bd1aa-cea2-433a-ba8f-ec87e384ac75 is now active with session: 70b25287-1744-4c51-92cb-0132403b057f","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.641635777Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.641648855Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379,"session_id":"70b25287-1744-4c51-92cb-0132403b057f"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:29.64167887Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 194 len 40960 data: 4 4 4 5 5 6 6 5 5 4 02/30 Read block 216 len 12288 03/30 Read block 108 len 20480 04/30 Flush 05/30 Write block 490 len 40960 data: 3 4 4 6 5 6 6 6 6 6 06/30 Read block 185 len 28672 07/30 Write block 161 len 32768 data: 8 9 8 8 8 7 6 4 08/30 Write block 257 len 32768 data: 8 8 8 7 5 5 5 4 09/30 Write block 363 len 32768 data: 6 6 7 6 5 6 6 7 10/30 Read block 394 len 8192 11/30 Read block 453 len 28672 12/30 Write block 270 len 8192 data: 4 5 13/30 Read block 265 len 20480 14/30 Write block 322 len 36864 data: 2 2 2 2 3 3 3 3 3 15/30 Write block 511 len 16384 data: 2 4 5 5 16/30 Write block 127 len 28672 data: 6 4 4 5 4 4 3 17/30 Read block 35 len 24576 18/30 Read block 105 len 36864 19/30 Read block 108 len 32768 20/30 Flush 21/30 Write block 469 len 20480 data: 5 5 5 5 5 22/30 Read block 516 len 28672 23/30 Read block 177 len 4096 24/30 Read block 217 len 4096 25/30 Flush 26/30 Write block 19 len 28672 data: 7 6 7 6 6 6 4 27/30 Read block 571 len 32768 28/30 Read block 278 len 28672 29/30 Read block 325 len 8192 30/30 Read block 480 len 8192 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 7 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 2 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 4 5 5 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:32:29.662552173Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1379} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 67 67 67 T T T 1 020-039 47 47 47 67 67 67 T T T 2 040-059 41 41 41 55 55 55 F F F 3 060-079 46 46 46 63 63 63 F F F 4 080-099 46 46 46 64 64 64 F F F 5 100-119 42 42 42 57 57 57 F F F 6 120-139 48 48 48 69 69 69 F F F 7 140-159 44 44 44 61 61 61 F F F 8 160-179 48 48 48 69 69 69 F F F 9 180-199 48 48 48 68 68 68 F F F 10 200-219 48 48 48 68 68 68 F F F 11 220-239 46 46 46 66 66 66 F F F 12 240-259 48 48 48 69 69 69 F F F 13 260-279 48 48 48 69 69 69 F F F 14 280-299 45 45 45 62 62 62 F F F 15 300-319 42 42 42 56 56 56 F F F 16 320-339 48 48 48 69 69 69 F F F 17 340-359 45 45 45 62 62 62 F F F 18 360-379 48 48 48 69 69 69 F F F 19 380-399 47 47 47 67 67 67 F F F 20 400-419 46 46 46 63 63 63 F F F 21 420-439 38 38 38 52 52 52 F F F 22 440-459 45 45 45 62 62 62 F F F 23 460-479 48 48 48 70 70 70 F F F 24 480-499 48 48 48 69 69 69 F F F 25 500-519 48 48 48 69 69 69 F F F 26 520-539 47 47 47 67 67 67 F F F 27 540-559 46 46 46 64 64 64 F F F 28 560-579 47 47 47 67 67 67 F F F 29 580-599 43 43 43 58 58 58 F F F Max gen: 48, Max flush: 70 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 71 71 71 F F F 1 020-039 49 49 49 71 71 71 F F F 2 040-059 41 41 41 55 55 55 F F F 3 060-079 46 46 46 63 63 63 F F F 4 080-099 46 46 46 64 64 64 F F F 5 100-119 42 42 42 57 57 57 F F F 6 120-139 48 48 48 69 69 69 F F F 7 140-159 44 44 44 61 61 61 F F F 8 160-179 48 48 48 69 69 69 F F F 9 180-199 48 48 48 68 68 68 F F F 10 200-219 48 48 48 68 68 68 F F F 11 220-239 46 46 46 66 66 66 F F F 12 240-259 48 48 48 69 69 69 F F F 13 260-279 48 48 48 69 69 69 F F F 14 280-299 45 45 45 62 62 62 F F F 15 300-319 42 42 42 56 56 56 F F F 16 320-339 48 48 48 69 69 69 F F F 17 340-359 45 45 45 62 62 62 F F F 18 360-379 48 48 48 69 69 69 F F F 19 380-399 47 47 47 67 67 67 F F F 20 400-419 46 46 46 63 63 63 F F F 21 420-439 38 38 38 52 52 52 F F F 22 440-459 45 45 45 62 62 62 F F F 23 460-479 48 48 48 70 70 70 F F F 24 480-499 48 48 48 69 69 69 F F F 25 500-519 48 48 48 69 69 69 F F F 26 520-539 47 47 47 67 67 67 F F F 27 540-559 46 46 46 64 64 64 F F F 28 560-579 47 47 47 67 67 67 F F F 29 580-599 43 43 43 58 58 58 F F F Max gen: 49, Max flush: 71 Begin loop 25 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.911641306Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.95537677Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.955988631Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.956022386Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.956038623Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.956053721Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387} {"msg":"Crucible 7398aa0c-66e6-4078-a303-429e710c86be has session id: 0d20ea65-ff91-4d96-8696-0154d566aeae","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.956462771Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"Upstairs opts: Upstairs UUID: 7398aa0c-66e6-4078-a303-429e710c86be, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.956512254Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"Crucible stats registered with UUID: 7398aa0c-66e6-4078-a303-429e710c86be","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.95653801Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.986828737Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.986863283Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387} {"msg":"7398aa0c-66e6-4078-a303-429e710c86be active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.986894558Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.986917784Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"client":"0","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.986959298Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"client":"0","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.986995633Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"client":"1","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.987030388Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"client":"1","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.987057504Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"client":"2","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.98708736Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"client":"2","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.987115736Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"":"io task","client":"2","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.987144091Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"":"io task","client":"0","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.987170068Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"":"io task","client":"1","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"ds_connection connected from Ok(127.0.0.1:62635)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.987195654Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"":"io task","client":"2","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"ds_connection connected from Ok(127.0.0.1:36864)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.98729067Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"":"io task","client":"0","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"ds_connection connected from Ok(127.0.0.1:49812)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.987321896Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"":"io task","client":"1","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.987956983Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"client":"1","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.987985349Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"client":"1","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.988017484Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"client":"0","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.988032822Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"client":"0","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.98810979Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"client":"2","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.988140636Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"client":"2","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.988530009Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"client":"0","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.988562224Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"client":"1","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.98886917Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"client":"2","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.989149079Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"client":"1","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.989177125Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"client":"1","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.989192303Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.98927605Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"client":"0","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.989298787Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"client":"0","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.989313345Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.989463433Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"client":"2","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.98948209Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"client":"2","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"[0]R flush_numbers[0..12]: [71, 71, 55, 63, 64, 57, 69, 61, 69, 68, 68, 66]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.989496268Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"":"downstairs","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"[0]R generation[0..12]: [49, 49, 41, 46, 46, 42, 48, 44, 48, 48, 48, 46]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.989510106Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"":"downstairs","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.989524334Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"":"downstairs","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"[1]R flush_numbers[0..12]: [71, 71, 55, 63, 64, 57, 69, 61, 69, 68, 68, 66]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.989538232Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"":"downstairs","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"[1]R generation[0..12]: [49, 49, 41, 46, 46, 42, 48, 44, 48, 48, 48, 46]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.98955177Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"":"downstairs","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.989565938Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"":"downstairs","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"[2]R flush_numbers[0..12]: [71, 71, 55, 63, 64, 57, 69, 61, 69, 68, 68, 66]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.989579586Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"":"downstairs","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"[2]R generation[0..12]: [49, 49, 41, 46, 46, 42, 48, 44, 48, 48, 48, 46]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.989593154Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"":"downstairs","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.989607142Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"":"downstairs","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.98962034Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"":"downstairs","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.989636638Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"":"downstairs","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"Next flush: 72","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:32:33.989649796Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"":"downstairs","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.989668533Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"":"downstairs","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.989681641Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.989693729Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"client":"0","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.989707167Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"client":"1","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.989720305Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"client":"2","":"downstairs","session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.989733464Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.989745962Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"7398aa0c-66e6-4078-a303-429e710c86be is now active with session: 0d20ea65-ff91-4d96-8696-0154d566aeae","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.98975899Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.989771248Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387,"session_id":"0d20ea65-ff91-4d96-8696-0154d566aeae"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:33.989783576Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 315 len 28672 data: 5 4 4 4 3 3 2 02/30 Write block 540 len 16384 data: 3 4 7 7 03/30 Flush 04/30 Write block 126 len 36864 data: 6 7 5 5 6 5 5 4 4 05/30 Write block 213 len 36864 data: 4 5 4 4 4 4 4 4 6 06/30 Read block 101 len 8192 07/30 Write block 170 len 40960 data: 4 4 3 2 3 4 6 5 5 5 08/30 Flush 09/30 Write block 117 len 20480 data: 5 8 10 10 9 10/30 Read block 59 len 32768 11/30 Write block 310 len 8192 data: 5 5 12/30 Read block 133 len 20480 13/30 Read block 545 len 20480 14/30 Read block 480 len 8192 15/30 Read block 592 len 20480 16/30 Write block 504 len 24576 data: 5 4 4 4 3 3 17/30 Read block 501 len 16384 18/30 Write block 208 len 28672 data: 3 2 2 3 3 5 6 19/30 Read block 36 len 12288 20/30 Read block 343 len 28672 21/30 Read block 81 len 12288 22/30 Read block 510 len 32768 23/30 Write block 334 len 24576 data: 5 4 4 4 5 7 24/30 Write block 147 len 12288 data: 6 4 5 25/30 Read block 442 len 4096 26/30 Flush 27/30 Write block 288 len 36864 data: 4 4 2 2 3 4 5 4 5 28/30 Read block 12 len 40960 29/30 Read block 582 len 4096 30/30 Read block 299 len 20480 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 9 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 4 3 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:32:34.010084812Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1387} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 71 71 71 F F F 1 020-039 49 49 49 71 71 71 F F F 2 040-059 41 41 41 55 55 55 F F F 3 060-079 46 46 46 63 63 63 F F F 4 080-099 46 46 46 64 64 64 F F F 5 100-119 50 50 50 74 74 74 F F F 6 120-139 50 50 50 74 74 74 F F F 7 140-159 50 50 50 74 74 74 F F F 8 160-179 50 50 50 73 73 73 F F F 9 180-199 48 48 48 68 68 68 F F F 10 200-219 50 50 50 74 74 74 F F F 11 220-239 50 50 50 73 73 73 F F F 12 240-259 48 48 48 69 69 69 F F F 13 260-279 48 48 48 69 69 69 F F F 14 280-299 45 45 45 62 62 62 T T T 15 300-319 50 50 50 74 74 74 F F F 16 320-339 50 50 50 74 74 74 F F F 17 340-359 45 45 45 62 62 62 F F F 18 360-379 48 48 48 69 69 69 F F F 19 380-399 47 47 47 67 67 67 F F F 20 400-419 46 46 46 63 63 63 F F F 21 420-439 38 38 38 52 52 52 F F F 22 440-459 45 45 45 62 62 62 F F F 23 460-479 48 48 48 70 70 70 F F F 24 480-499 48 48 48 69 69 69 F F F 25 500-519 50 50 50 74 74 74 F F F 26 520-539 47 47 47 67 67 67 F F F 27 540-559 50 50 50 72 72 72 F F F 28 560-579 47 47 47 67 67 67 F F F 29 580-599 43 43 43 58 58 58 F F F Max gen: 50, Max flush: 74 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 71 71 71 F F F 1 020-039 49 49 49 71 71 71 F F F 2 040-059 41 41 41 55 55 55 F F F 3 060-079 46 46 46 63 63 63 F F F 4 080-099 46 46 46 64 64 64 F F F 5 100-119 50 50 50 74 74 74 F F F 6 120-139 50 50 50 74 74 74 F F F 7 140-159 50 50 50 74 74 74 F F F 8 160-179 50 50 50 73 73 73 F F F 9 180-199 48 48 48 68 68 68 F F F 10 200-219 50 50 50 74 74 74 F F F 11 220-239 50 50 50 73 73 73 F F F 12 240-259 48 48 48 69 69 69 F F F 13 260-279 48 48 48 69 69 69 F F F 14 280-299 51 51 51 75 75 75 F F F 15 300-319 50 50 50 74 74 74 F F F 16 320-339 50 50 50 74 74 74 F F F 17 340-359 45 45 45 62 62 62 F F F 18 360-379 48 48 48 69 69 69 F F F 19 380-399 47 47 47 67 67 67 F F F 20 400-419 46 46 46 63 63 63 F F F 21 420-439 38 38 38 52 52 52 F F F 22 440-459 45 45 45 62 62 62 F F F 23 460-479 48 48 48 70 70 70 F F F 24 480-499 48 48 48 69 69 69 F F F 25 500-519 50 50 50 74 74 74 F F F 26 520-539 47 47 47 67 67 67 F F F 27 540-559 50 50 50 72 72 72 F F F 28 560-579 47 47 47 67 67 67 F F F 29 580-599 43 43 43 58 58 58 F F F Max gen: 51, Max flush: 75 Begin loop 26 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.255838332Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.298959066Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.299659434Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.299693799Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.299708496Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.299723014Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395} {"msg":"Crucible d044fed5-a73b-4899-9559-5ce89e928652 has session id: 41c45d7a-f77a-4c99-b032-69f760be5998","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.300218392Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"Upstairs opts: Upstairs UUID: d044fed5-a73b-4899-9559-5ce89e928652, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.300267505Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"Crucible stats registered with UUID: d044fed5-a73b-4899-9559-5ce89e928652","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.300293221Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.33125946Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.331296634Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395} {"msg":"d044fed5-a73b-4899-9559-5ce89e928652 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.331331359Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.331352146Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"client":"0","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.33139492Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"client":"0","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.331432674Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"client":"1","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.33146278Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"client":"1","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.331479128Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"client":"2","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.331504394Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"client":"2","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.33153067Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"":"io task","client":"2","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.331560086Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"":"io task","client":"0","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.331590621Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"":"io task","client":"1","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"ds_connection connected from Ok(127.0.0.1:58870)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.331620867Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"":"io task","client":"2","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"ds_connection connected from Ok(127.0.0.1:48230)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.331648043Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"":"io task","client":"0","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"ds_connection connected from Ok(127.0.0.1:58044)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.331714403Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"":"io task","client":"1","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.33235476Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"client":"2","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.332390964Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"client":"2","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.332413911Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"client":"0","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.332444736Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"client":"0","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.33262678Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"client":"1","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.332660595Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"client":"1","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.333100801Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"client":"2","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.333142375Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"client":"0","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.333211575Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"client":"1","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.333787231Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"client":"1","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.333813117Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"client":"1","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.333827325Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.333839593Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"client":"2","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.333853021Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"client":"2","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.333866209Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.333878347Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"client":"0","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.333891515Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"client":"0","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"[0]R flush_numbers[0..12]: [71, 71, 55, 63, 64, 74, 74, 74, 73, 68, 74, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.333905373Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"":"downstairs","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"[0]R generation[0..12]: [49, 49, 41, 46, 46, 50, 50, 50, 50, 48, 50, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.333922521Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"":"downstairs","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.333937059Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"":"downstairs","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"[1]R flush_numbers[0..12]: [71, 71, 55, 63, 64, 74, 74, 74, 73, 68, 74, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.333950847Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"":"downstairs","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"[1]R generation[0..12]: [49, 49, 41, 46, 46, 50, 50, 50, 50, 48, 50, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.333964455Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"":"downstairs","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.333978453Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"":"downstairs","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R flush_numbers[0..12]: [71, 71, 55, 63, 64, 74, 74, 74, 73, 68, 74, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.334001339Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"":"downstairs","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"[2]R generation[0..12]: [49, 49, 41, 46, 46, 50, 50, 50, 50, 48, 50, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.334024856Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"":"downstairs","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.334039994Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"":"downstairs","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.334053362Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"":"downstairs","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.33406657Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"":"downstairs","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"Next flush: 76","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.334079668Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"":"downstairs","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.334092536Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"":"downstairs","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.334106044Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.334118192Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"client":"0","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.3341338Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"client":"1","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.334148178Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"client":"2","":"downstairs","session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.334161926Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.334174484Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"d044fed5-a73b-4899-9559-5ce89e928652 is now active with session: 41c45d7a-f77a-4c99-b032-69f760be5998","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.334187422Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.33419957Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395,"session_id":"41c45d7a-f77a-4c99-b032-69f760be5998"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:38.334212329Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 69 len 8192 data: 4 6 02/30 Write block 493 len 36864 data: 7 6 7 7 7 7 7 7 6 03/30 Read block 216 len 16384 04/30 Read block 146 len 12288 05/30 Flush 06/30 Flush 07/30 Write block 245 len 28672 data: 8 6 7 7 5 7 6 08/30 Read block 143 len 32768 09/30 Read block 257 len 8192 10/30 Write block 205 len 40960 data: 3 4 5 4 3 3 4 4 6 7 11/30 Write block 476 len 8192 data: 6 5 12/30 Flush 13/30 Write block 550 len 36864 data: 8 8 9 7 5 5 5 5 5 14/30 Write block 403 len 32768 data: 3 4 4 4 3 4 4 5 15/30 Read block 520 len 8192 16/30 Read block 317 len 20480 17/30 Read block 134 len 20480 18/30 Write block 587 len 12288 data: 6 7 6 19/30 Read block 496 len 16384 20/30 Write block 516 len 4096 data: 5 21/30 Flush 22/30 Write block 62 len 12288 data: 7 6 8 23/30 Read block 566 len 24576 24/30 Write block 285 len 28672 data: 5 4 4 5 5 3 3 25/30 Write block 285 len 36864 data: 6 5 5 6 6 4 4 4 5 26/30 Write block 271 len 32768 data: 6 3 3 3 4 3 3 3 27/30 Read block 381 len 32768 28/30 Write block 324 len 40960 data: 3 3 4 4 4 4 4 3 3 4 29/30 Write block 349 len 12288 data: 4 4 6 30/30 Write block 152 len 32768 data: 4 3 4 4 4 3 5 6 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 3 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:32:38.352998486Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1395} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 71 71 71 F F F 1 020-039 49 49 49 71 71 71 F F F 2 040-059 41 41 41 55 55 55 F F F 3 060-079 52 52 52 76 76 76 T T T 4 080-099 46 46 46 64 64 64 F F F 5 100-119 50 50 50 74 74 74 F F F 6 120-139 50 50 50 74 74 74 F F F 7 140-159 50 50 50 74 74 74 T T T 8 160-179 50 50 50 73 73 73 F F F 9 180-199 48 48 48 68 68 68 F F F 10 200-219 52 52 52 78 78 78 F F F 11 220-239 50 50 50 73 73 73 F F F 12 240-259 52 52 52 78 78 78 F F F 13 260-279 48 48 48 69 69 69 T T T 14 280-299 51 51 51 75 75 75 T T T 15 300-319 50 50 50 74 74 74 F F F 16 320-339 50 50 50 74 74 74 T T T 17 340-359 45 45 45 62 62 62 T T T 18 360-379 48 48 48 69 69 69 F F F 19 380-399 47 47 47 67 67 67 F F F 20 400-419 52 52 52 79 79 79 F F F 21 420-439 38 38 38 52 52 52 F F F 22 440-459 45 45 45 62 62 62 F F F 23 460-479 52 52 52 78 78 78 F F F 24 480-499 52 52 52 76 76 76 F F F 25 500-519 52 52 52 79 79 79 F F F 26 520-539 47 47 47 67 67 67 F F F 27 540-559 52 52 52 79 79 79 F F F 28 560-579 47 47 47 67 67 67 F F F 29 580-599 52 52 52 79 79 79 F F F Max gen: 52, Max flush: 79 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 71 71 71 F F F 1 020-039 49 49 49 71 71 71 F F F 2 040-059 41 41 41 55 55 55 F F F 3 060-079 53 53 53 80 80 80 F F F 4 080-099 46 46 46 64 64 64 F F F 5 100-119 50 50 50 74 74 74 F F F 6 120-139 50 50 50 74 74 74 F F F 7 140-159 53 53 53 80 80 80 F F F 8 160-179 50 50 50 73 73 73 F F F 9 180-199 48 48 48 68 68 68 F F F 10 200-219 52 52 52 78 78 78 F F F 11 220-239 50 50 50 73 73 73 F F F 12 240-259 52 52 52 78 78 78 F F F 13 260-279 53 53 53 80 80 80 F F F 14 280-299 53 53 53 80 80 80 F F F 15 300-319 50 50 50 74 74 74 F F F 16 320-339 53 53 53 80 80 80 F F F 17 340-359 53 53 53 80 80 80 F F F 18 360-379 48 48 48 69 69 69 F F F 19 380-399 47 47 47 67 67 67 F F F 20 400-419 52 52 52 79 79 79 F F F 21 420-439 38 38 38 52 52 52 F F F 22 440-459 45 45 45 62 62 62 F F F 23 460-479 52 52 52 78 78 78 F F F 24 480-499 52 52 52 76 76 76 F F F 25 500-519 52 52 52 79 79 79 F F F 26 520-539 47 47 47 67 67 67 F F F 27 540-559 52 52 52 79 79 79 F F F 28 560-579 47 47 47 67 67 67 F F F 29 580-599 52 52 52 79 79 79 F F F Max gen: 53, Max flush: 80 Begin loop 27 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.626475829Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.67132625Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.672224019Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.672276151Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.672299488Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.672324194Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403} {"msg":"Crucible fb56af22-827b-4107-ba9d-2c33ccb7018b has session id: 159285e3-6d71-476e-aeb7-b9531ac58b34","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.672756041Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"Upstairs opts: Upstairs UUID: fb56af22-827b-4107-ba9d-2c33ccb7018b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.672802414Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"Crucible stats registered with UUID: fb56af22-827b-4107-ba9d-2c33ccb7018b","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.67283025Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.703394257Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.70344481Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403} {"msg":"fb56af22-827b-4107-ba9d-2c33ccb7018b active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.703461058Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.703477505Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"client":"0","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.703505831Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"client":"0","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.703531777Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"client":"1","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.703569392Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"client":"1","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.703605047Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"client":"2","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.703640992Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"client":"2","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.703676296Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"":"io task","client":"2","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.703709182Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"":"io task","client":"0","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.703735838Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"":"io task","client":"1","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"ds_connection connected from Ok(127.0.0.1:46727)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.703762274Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"":"io task","client":"0","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"ds_connection connected from Ok(127.0.0.1:33675)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.703973403Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"":"io task","client":"1","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"ds_connection connected from Ok(127.0.0.1:34298)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.704164955Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"":"io task","client":"2","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.704553918Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"client":"0","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.704582464Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"client":"0","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.704737791Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"client":"1","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.704782475Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"client":"1","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.704814Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"client":"2","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.704842276Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"client":"2","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.705282282Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"client":"0","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.705318197Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"client":"1","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.705552802Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"client":"2","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.705898252Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"client":"0","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.705938886Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"client":"0","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.705968812Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.706019084Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"client":"1","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.706054539Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"client":"1","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.706082625Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.70611447Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"client":"2","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.706140907Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"client":"2","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"[0]R flush_numbers[0..12]: [71, 71, 55, 80, 64, 74, 74, 80, 73, 68, 78, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.706167303Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"":"downstairs","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"[0]R generation[0..12]: [49, 49, 41, 53, 46, 50, 50, 53, 50, 48, 52, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.706192539Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"":"downstairs","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.706218735Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"":"downstairs","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"[1]R flush_numbers[0..12]: [71, 71, 55, 80, 64, 74, 74, 80, 73, 68, 78, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.706243552Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"":"downstairs","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"[1]R generation[0..12]: [49, 49, 41, 53, 46, 50, 50, 53, 50, 48, 52, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.706270538Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"":"downstairs","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.706295614Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"":"downstairs","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"[2]R flush_numbers[0..12]: [71, 71, 55, 80, 64, 74, 74, 80, 73, 68, 78, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.70632148Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"":"downstairs","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"[2]R generation[0..12]: [49, 49, 41, 53, 46, 50, 50, 53, 50, 48, 52, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.706348776Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"":"downstairs","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.706374302Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"":"downstairs","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.706401469Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"":"downstairs","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.706437633Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"":"downstairs","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"Next flush: 81","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.706467119Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"":"downstairs","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.706495645Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"":"downstairs","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.706523481Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.706548707Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"client":"0","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.706566734Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"client":"1","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.706580462Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"client":"2","":"downstairs","session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.70659392Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.706606539Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"fb56af22-827b-4107-ba9d-2c33ccb7018b is now active with session: 159285e3-6d71-476e-aeb7-b9531ac58b34","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.706622556Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:42.706634964Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403,"session_id":"159285e3-6d71-476e-aeb7-b9531ac58b34"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 level":30,"time":"2025-01-03T08:32:42.706650862Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 46 len 36864 data: 7 5 6 7 7 6 7 5 5 02/30 Read block 391 len 20480 03/30 Read block 591 len 24576 04/30 Read block 81 len 12288 05/30 Read block 415 len 32768 06/30 Read block 186 len 16384 07/30 Write block 353 len 4096 data: 4 08/30 Read block 278 len 36864 09/30 Read block 112 len 8192 10/30 Read block 492 len 32768 11/30 Read block 288 len 4096 12/30 Write block 525 len 8192 data: 9 10 13/30 Read block 176 len 24576 14/30 Read block 311 len 24576 15/30 Read block 0 len 4096 16/30 Read block 305 len 4096 17/30 Read block 217 len 20480 18/30 Write block 551 len 12288 data: 9 10 8 19/30 Read block 282 len 32768 20/30 Read block 29 len 40960 21/30 Write block 79 len 32768 data: 6 7 7 5 5 4 7 4 22/30 Write block 480 len 12288 data: 5 4 4 23/30 Write block 406 len 40960 data: 5 4 5 5 6 8 9 10 13 12 24/30 Flush 25/30 Write block 432 len 20480 data: 4 2 2 5 5 26/30 Read block 287 len 12288 27/30 Write block 361 len 4096 data: 6 28/30 Read block 499 len 36864 29/30 Write block 30 len 8192 data: 5 4 30/30 Read block 391 len 32768 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 5 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 2 Done Done Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 6 5 4 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:32:42.725961342Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1403} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 71 71 71 F F F 1 020-039 49 49 49 71 71 71 T T T 2 040-059 54 54 54 81 81 81 F F F 3 060-079 54 54 54 81 81 81 F F F 4 080-099 54 54 54 81 81 81 F F F 5 100-119 50 50 50 74 74 74 F F F 6 120-139 50 50 50 74 74 74 F F F 7 140-159 53 53 53 80 80 80 F F F 8 160-179 50 50 50 73 73 73 F F F 9 180-199 48 48 48 68 68 68 F F F 10 200-219 52 52 52 78 78 78 F F F 11 220-239 50 50 50 73 73 73 F F F 12 240-259 52 52 52 78 78 78 F F F 13 260-279 53 53 53 80 80 80 F F F 14 280-299 53 53 53 80 80 80 F F F 15 300-319 50 50 50 74 74 74 F F F 16 320-339 53 53 53 80 80 80 F F F 17 340-359 54 54 54 81 81 81 F F F 18 360-379 48 48 48 69 69 69 T T T 19 380-399 47 47 47 67 67 67 F F F 20 400-419 54 54 54 81 81 81 F F F 21 420-439 38 38 38 52 52 52 T T T 22 440-459 45 45 45 62 62 62 F F F 23 460-479 52 52 52 78 78 78 F F F 24 480-499 54 54 54 81 81 81 F F F 25 500-519 52 52 52 79 79 79 F F F 26 520-539 54 54 54 81 81 81 F F F 27 540-559 54 54 54 81 81 81 F F F 28 560-579 47 47 47 67 67 67 F F F 29 580-599 52 52 52 79 79 79 F F F Max gen: 54, Max flush: 81 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 71 71 71 F F F 1 020-039 55 55 55 82 82 82 F F F 2 040-059 54 54 54 81 81 81 F F F 3 060-079 54 54 54 81 81 81 F F F 4 080-099 54 54 54 81 81 81 F F F 5 100-119 50 50 50 74 74 74 F F F 6 120-139 50 50 50 74 74 74 F F F 7 140-159 53 53 53 80 80 80 F F F 8 160-179 50 50 50 73 73 73 F F F 9 180-199 48 48 48 68 68 68 F F F 10 200-219 52 52 52 78 78 78 F F F 11 220-239 50 50 50 73 73 73 F F F 12 240-259 52 52 52 78 78 78 F F F 13 260-279 53 53 53 80 80 80 F F F 14 280-299 53 53 53 80 80 80 F F F 15 300-319 50 50 50 74 74 74 F F F 16 320-339 53 53 53 80 80 80 F F F 17 340-359 54 54 54 81 81 81 F F F 18 360-379 55 55 55 82 82 82 F F F 19 380-399 47 47 47 67 67 67 F F F 20 400-419 54 54 54 81 81 81 F F F 21 420-439 55 55 55 82 82 82 F F F 22 440-459 45 45 45 62 62 62 F F F 23 460-479 52 52 52 78 78 78 F F F 24 480-499 54 54 54 81 81 81 F F F 25 500-519 52 52 52 79 79 79 F F F 26 520-539 54 54 54 81 81 81 F F F 27 540-559 54 54 54 81 81 81 F F F 28 560-579 47 47 47 67 67 67 F F F 29 580-599 52 52 52 79 79 79 F F F Max gen: 55, Max flush: 82 Begin loop 28 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:46.991537837Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:32:47.034011715Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.034370483Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.06603128Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.066592078Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.066627093Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.06664538Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.066660788Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411} {"msg":"Crucible 83843894-186b-4db1-8769-b68142a53075 has session id: 1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.067132109Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"Upstairs opts: Upstairs UUID: 83843894-186b-4db1-8769-b68142a53075, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.067176553Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"Crucible stats registered with UUID: 83843894-186b-4db1-8769-b68142a53075","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.067202219Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.097162544Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.097208448Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411} {"msg":"83843894-186b-4db1-8769-b68142a53075 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.097245682Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.097277198Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"client":"0","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.097315762Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"client":"0","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.097347367Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"client":"1","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.097376083Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"client":"1","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.097405309Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"client":"2","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.097437104Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"client":"2","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.09746754Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"":"io task","client":"2","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.097494216Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"":"io task","client":"0","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.097523182Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"":"io task","client":"1","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"ds_connection connected from Ok(127.0.0.1:56648)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.097552227Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"":"io task","client":"1","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"ds_connection connected from Ok(127.0.0.1:59234)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.097578243Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"":"io task","client":"0","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"ds_connection connected from Ok(127.0.0.1:51859)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.097770905Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"":"io task","client":"2","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.098379857Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"client":"0","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.098409492Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"client":"0","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.09842653Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"client":"2","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.098441068Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"client":"2","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.098456995Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"client":"1","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.098481422Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"client":"1","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.098997266Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"client":"1","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.099199807Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"client":"2","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.099285754Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"client":"0","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.099613796Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"client":"1","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.099642872Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"client":"1","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.09965784Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.099670848Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"client":"2","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.099684706Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"client":"2","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.099698314Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.099799519Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"client":"0","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.099819216Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"client":"0","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"[0]R flush_numbers[0..12]: [71, 82, 81, 81, 81, 74, 74, 80, 73, 68, 78, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.099833294Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"":"downstairs","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"[0]R generation[0..12]: [49, 55, 54, 54, 54, 50, 50, 53, 50, 48, 52, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.099847502Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"":"downstairs","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.0998616Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"":"downstairs","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"[1]R flush_numbers[0..12]: [71, 82, 81, 81, 81, 74, 74, 80, 73, 68, 78, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.099875868Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"":"downstairs","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"[1]R generation[0..12]: [49, 55, 54, 54, 54, 50, 50, 53, 50, 48, 52, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.099889726Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"":"downstairs","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.099904014Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"":"downstairs","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"[2]R flush_numbers[0..12]: [71, 82, 81, 81, 81, 74, 74, 80, 73, 68, 78, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.099917982Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"":"downstairs","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"[2]R generation[0..12]: [49, 55, 54, 54, 54, 50, 50, 53, 50, 48, 52, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.099936499Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"":"downstairs","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.099951437Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"":"downstairs","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.099964625Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"":"downstairs","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.099978083Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"":"downstairs","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"Next flush: 83","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.099991241Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.100021897Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"":"downstairs","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.100035295Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.100047313Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"client":"0","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.100060651Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"client":"1","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.100073899Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"client":"2","":"downstairs","session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.100087187Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.100099435Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"83843894-186b-4db1-8769-b68142a53075 is now active with session: 1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.100112284Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.100124472Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411,"session_id":"1dae54d4-0cc7-4d7b-99ef-b1e88fcc932f"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:47.10013685Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 8 len 24576 data: 4 3 3 3 3 4 03/30 Flush 04/30 Write block 357 len 4096 data: 4 05/30 Flush 06/30 Flush 07/30 Write block 457 len 8192 data: 5 6 08/30 Read block 237 len 28672 09/30 Write block 42 len 8192 data: 6 5 10/30 Read block 428 len 24576 11/30 Write block 251 len 16384 data: 7 9 8 8 12/30 Write block 518 len 8192 data: 5 4 13/30 Read block 356 len 28672 14/30 Read block 16 len 8192 15/30 Flush 16/30 Write block 568 len 24576 data: 4 4 3 3 3 4 17/30 Read block 362 len 28672 18/30 Read block 481 len 12288 19/30 Read block 388 len 12288 20/30 Write block 566 len 24576 data: 4 4 5 5 4 4 21/30 Read block 155 len 8192 22/30 Read block 359 len 36864 23/30 Write block 77 len 8192 data: 6 6 24/30 Read block 410 len 12288 25/30 Read block 504 len 32768 26/30 Read block 153 len 28672 27/30 Write block 558 len 12288 data: 6 4 3 28/30 Write block 587 len 20480 data: 7 8 7 5 5 29/30 Write block 554 len 12288 data: 6 6 6 30/30 Read block 507 len 36864 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 6 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 14 15 13 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:32:47.117842325Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1411} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 84 84 84 F F F 1 020-039 55 55 55 82 82 82 F F F 2 040-059 56 56 56 87 87 87 F F F 3 060-079 54 54 54 81 81 81 T T T 4 080-099 54 54 54 81 81 81 F F F 5 100-119 50 50 50 74 74 74 F F F 6 120-139 50 50 50 74 74 74 F F F 7 140-159 53 53 53 80 80 80 F F F 8 160-179 50 50 50 73 73 73 F F F 9 180-199 48 48 48 68 68 68 F F F 10 200-219 52 52 52 78 78 78 F F F 11 220-239 50 50 50 73 73 73 F F F 12 240-259 56 56 56 87 87 87 F F F 13 260-279 53 53 53 80 80 80 F F F 14 280-299 53 53 53 80 80 80 F F F 15 300-319 50 50 50 74 74 74 F F F 16 320-339 53 53 53 80 80 80 F F F 17 340-359 56 56 56 85 85 85 F F F 18 360-379 55 55 55 82 82 82 F F F 19 380-399 47 47 47 67 67 67 F F F 20 400-419 54 54 54 81 81 81 F F F 21 420-439 55 55 55 82 82 82 F F F 22 440-459 56 56 56 87 87 87 F F F 23 460-479 52 52 52 78 78 78 F F F 24 480-499 54 54 54 81 81 81 F F F 25 500-519 56 56 56 87 87 87 F F F 26 520-539 54 54 54 81 81 81 F F F 27 540-559 54 54 54 81 81 81 T T T 28 560-579 47 47 47 67 67 67 T T T 29 580-599 52 52 52 79 79 79 T T T Max gen: 56, Max flush: 87 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 84 84 84 F F F 1 020-039 55 55 55 82 82 82 F F F 2 040-059 56 56 56 87 87 87 F F F 3 060-079 57 57 57 88 88 88 F F F 4 080-099 54 54 54 81 81 81 F F F 5 100-119 50 50 50 74 74 74 F F F 6 120-139 50 50 50 74 74 74 F F F 7 140-159 53 53 53 80 80 80 F F F 8 160-179 50 50 50 73 73 73 F F F 9 180-199 48 48 48 68 68 68 F F F 10 200-219 52 52 52 78 78 78 F F F 11 220-239 50 50 50 73 73 73 F F F 12 240-259 56 56 56 87 87 87 F F F 13 260-279 53 53 53 80 80 80 F F F 14 280-299 53 53 53 80 80 80 F F F 15 300-319 50 50 50 74 74 74 F F F 16 320-339 53 53 53 80 80 80 F F F 17 340-359 56 56 56 85 85 85 F F F 18 360-379 55 55 55 82 82 82 F F F 19 380-399 47 47 47 67 67 67 F F F 20 400-419 54 54 54 81 81 81 F F F 21 420-439 55 55 55 82 82 82 F F F 22 440-459 56 56 56 87 87 87 F F F 23 460-479 52 52 52 78 78 78 F F F 24 480-499 54 54 54 81 81 81 F F F 25 500-519 56 56 56 87 87 87 F F F 26 520-539 54 54 54 81 81 81 F F F 27 540-559 57 57 57 88 88 88 F F F 28 560-579 57 57 57 88 88 88 F F F 29 580-599 57 57 57 88 88 88 F F F Max gen: 57, Max flush: 88 Begin loop 29 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.409293963Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.453340521Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.453974929Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.454019192Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.4540374Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.454056237Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419} {"msg":"Crucible f592eb5b-0776-4983-b6b3-d12b74cf1103 has session id: 759c16cd-7819-4e65-a627-a281c314376b","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.454574121Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"Upstairs opts: Upstairs UUID: f592eb5b-0776-4983-b6b3-d12b74cf1103, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.454684715Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"Crucible stats registered with UUID: f592eb5b-0776-4983-b6b3-d12b74cf1103","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.454707212Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.485577764Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.48560998Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419} {"msg":"f592eb5b-0776-4983-b6b3-d12b74cf1103 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.485639035Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.485656073Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"client":"0","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.485682759Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"client":"0","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.485701596Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"client":"1","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.485719104Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"client":"1","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.485739411Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"client":"2","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.485771316Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"client":"2","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.485804251Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"":"io task","client":"2","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.485833517Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"":"io task","client":"0","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.485852454Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"":"io task","client":"1","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"ds_connection connected from Ok(127.0.0.1:35896)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.48588235Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"":"io task","client":"0","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"ds_connection connected from Ok(127.0.0.1:50049)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.485922604Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"":"io task","client":"2","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"ds_connection connected from Ok(127.0.0.1:53688)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.485956129Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"":"io task","client":"1","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.486726397Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"client":"1","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.486755802Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"client":"1","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.486874545Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"client":"0","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.486897292Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"client":"0","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.48691121Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"client":"2","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.486924828Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"client":"2","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.487350306Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"client":"1","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.48739012Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"client":"0","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.487680367Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"client":"2","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.487912553Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"client":"0","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.48793767Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"client":"0","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.487953267Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.488019508Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"client":"1","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.488040315Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"client":"1","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.488054533Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.488161347Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"client":"2","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.488181394Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"client":"2","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"[0]R flush_numbers[0..12]: [84, 82, 87, 88, 81, 74, 74, 80, 73, 68, 78, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.488196552Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"":"downstairs","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"[0]R generation[0..12]: [56, 55, 56, 57, 54, 50, 50, 53, 50, 48, 52, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.48821045Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"":"downstairs","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.488224588Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"":"downstairs","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"[1]R flush_numbers[0..12]: [84, 82, 87, 88, 81, 74, 74, 80, 73, 68, 78, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.488238396Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"":"downstairs","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"[1]R generation[0..12]: [56, 55, 56, 57, 54, 50, 50, 53, 50, 48, 52, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.488251934Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"":"downstairs","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.488265852Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"":"downstairs","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"[2]R flush_numbers[0..12]: [84, 82, 87, 88, 81, 74, 74, 80, 73, 68, 78, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.48827952Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"":"downstairs","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"[2]R generation[0..12]: [56, 55, 56, 57, 54, 50, 50, 53, 50, 48, 52, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.488293148Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"":"downstairs","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.488307346Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"":"downstairs","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.488320614Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"":"downstairs","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.488336931Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"":"downstairs","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"Next flush: 89"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.488358238Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"":"downstairs","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.488378675Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"":"downstairs","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.488405951Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.488428858Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"client":"0","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.488453434Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"client":"1","":"downstairs","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.488479841Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"client":"2","":"downstairsDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ","session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.488510176Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.488534223Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"f592eb5b-0776-4983-b6b3-d12b74cf1103 is now active with session: 759c16cd-7819-4e65-a627-a281c314376b","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.48854991Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.488563308Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419,"session_id":"759c16cd-7819-4e65-a627-a281c314376b"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:51.488575996Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 129 len 28672 data: 6 7 6 6 5 5 5 02/30 Write block 341 len 24576 data: 8 7 5 6 7 6 03/30 Read block 141 len 8192 04/30 Read block 386 len 20480 05/30 Write block 321 len 28672 data: 3 3 3 4 4 5 5 06/30 Write block 442 len 40960 data: 6 5 5 5 5 5 5 5 5 5 07/30 Read block 164 len 28672 08/30 Read block 375 len 24576 09/30 Read block 96 len 32768 10/30 Write block 290 len 40960 data: 5 5 5 6 6 5 6 6 5 5 11/30 Read block 308 len 32768 12/30 Read block 132 len 36864 13/30 Flush 14/30 Write block 15 len 28672 data: 4 4 5 6 8 7 8 15/30 Write block 183 len 36864 data: 7 8 8 8 8 8 8 8 7 16/30 Write block 588 len 16384 data: 9 8 6 6 17/30 Read block 248 len 16384 18/30 Read block 177 len 36864 19/30 Read block 533 len 24576 20/30 Write block 564 len 24576 data: 6 4 5 5 6 6 21/30 Read block 277 len 4096 22/30 Flush 23/30 Write block 85 len 8192 data: 8 5 24/30 Read block 235 len 8192 25/30 Write block 13 len 28672 data: 5 5 5 5 6 7 9 26/30 Flush 27/30 Write block 416 len 24576 data: 11 10 8 8 6 6 28/30 Write block 273 len 8192 data: 4 4 29/30 Write block 161 len 8192 data: 9 10 30/30 Write block 122 len 8192 data: 8 9 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 6 Sent Sent Sent false 1027 Acked Write 2 Sent Sent Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 4 4 12 Done 0 0 0 0 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:32:51.507228543Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1419} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 91 91 91 F F F 1 020-039 58 58 58 90 90 90 F F F 2 040-059 56 56 56 87 87 87 F F F 3 060-079 57 57 57 88 88 88 F F F 4 080-099 58 58 58 91 91 91 F F F 5 100-119 50 50 50 74 74 74 F F F 6 120-139 58 58 58 89 89 89 T T T 7 140-159 53 53 53 80 80 80 F F F 8 160-179 50 50 50 73 73 73 T T T 9 180-199 58 58 58 90 90 90 F F F 10 200-219 52 52 52 78 78 78 F F F 11 220-239 50 50 50 73 73 73 F F F 12 240-259 56 56 56 87 87 87 F F F 13 260-279 53 53 53 80 80 80 T T T 14 280-299 58 58 58 89 89 89 F F F 15 300-319 50 50 50 74 74 74 F F F 16 320-339 58 58 58 89 89 89 F F F 17 340-359 58 58 58 89 89 89 F F F 18 360-379 55 55 55 82 82 82 F F F 19 380-399 47 47 47 67 67 67 F F F 20 400-419 54 54 54 81 81 81 T T T 21 420-439 55 55 55 82 82 82 T T T 22 440-459 58 58 58 89 89 89 F F F 23 460-479 52 52 52 78 78 78 F F F 24 480-499 54 54 54 81 81 81 F F F 25 500-519 56 56 56 87 87 87 F F F 26 520-539 54 54 54 81 81 81 F F F 27 540-559 57 57 57 88 88 88 F F F 28 560-579 58 58 58 90 90 90 F F F 29 580-599 58 58 58 90 90 90 F F F Max gen: 58, Max flush: 91 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 91 91 91 F F F 1 020-039 58 58 58 90 90 90 F F F 2 040-059 56 56 56 87 87 87 F F F 3 060-079 57 57 57 88 88 88 F F F 4 080-099 58 58 58 91 91 91 F F F 5 100-119 50 50 50 74 74 74 F F F 6 120-139 59 59 59 92 92 92 F F F 7 140-159 53 53 53 80 80 80 F F F 8 160-179 59 59 59 92 92 92 F F F 9 180-199 58 58 58 90 90 90 F F F 10 200-219 52 52 52 78 78 78 F F F 11 220-239 50 50 50 73 73 73 F F F 12 240-259 56 56 56 87 87 87 F F F 13 260-279 59 59 59 92 92 92 F F F 14 280-299 58 58 58 89 89 89 F F F 15 300-319 50 50 50 74 74 74 F F F 16 320-339 58 58 58 89 89 89 F F F 17 340-359 58 58 58 89 89 89 F F F 18 360-379 55 55 55 82 82 82 F F F 19 380-399 47 47 47 67 67 67 F F F 20 400-419 59 59 59 92 92 92 F F F 21 420-439 59 59 59 92 92 92 F F F 22 440-459 58 58 58 89 89 89 F F F 23 460-479 52 52 52 78 78 78 F F F 24 480-499 54 54 54 81 81 81 F F F 25 500-519 56 56 56 87 87 87 F F F 26 520-539 54 54 54 81 81 81 F F F 27 540-559 57 57 57 88 88 88 F F F 28 560-579 58 58 58 90 90 90 F F F 29 580-599 58 58 58 90 90 90 F F F Max gen: 59, Max flush: 92 Begin loop 30 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.771516827Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:32:55.814315088Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.814687753Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.846477961Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.847061306Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.847102Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.847116448Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.847131596Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427} {"msg":"Crucible f7f029a5-2ca5-4edc-85fa-88aba8f1b9cf has session id: 723aadc5-88b6-439e-b26a-dfacb8d65110","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.847617365Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"Upstairs opts: Upstairs UUID: f7f029a5-2ca5-4edc-85fa-88aba8f1b9cf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.847672507Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"Crucible stats registered with UUID: f7f029a5-2ca5-4edc-85fa-88aba8f1b9cf","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.847709672Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.878139468Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.878173273Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427} {"msg":"f7f029a5-2ca5-4edc-85fa-88aba8f1b9cf active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.87819986Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.878217027Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"client":"0","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.878246003Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"client":"0","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.87826407Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"client":"1","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.878281768Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"client":"1","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.878298795Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"client":"2","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.878316192Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"client":"2","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.878337569Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"":"io task","client":"2","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.878368925Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"":"io task","client":"0","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.87840129Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"":"io task","client":"1","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"ds_connection connected from Ok(127.0.0.1:52325)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.878433685Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"":"io task","client":"0","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"ds_connection connected from Ok(127.0.0.1:43908)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.878467001Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"":"io task","client":"1","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"ds_connection connected from Ok(127.0.0.1:53285)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.878500976Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"":"io task","client":"2","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.879346012Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"client":"2","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.879374648Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"client":"2","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.879389766Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"client":"1","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.879403684Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"client":"1","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.879542623Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"client":"0","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.879614503Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"client":"0","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.879970271Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"client":"2","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.880013615Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"client":"1","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.880161033Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"client":"0","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.880486875Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"client":"2","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.880513622Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"client":"2","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.88052854Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.880620976Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"client":"1","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.880640093Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"client":"1","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.880653571Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.88073187Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"client":"0","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.880753636Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"client":"0","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"[0]R flush_numbers[0..12]: [91, 90, 87, 88, 91, 74, 92, 80, 92, 90, 78, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.880768045Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"":"downstairs","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"[0]R generation[0..12]: [58, 58, 56, 57, 58, 50, 59, 53, 59, 58, 52, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.880782023Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"":"downstairs","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.88079617Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"":"downstairs","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"[1]R flush_numbers[0..12]: [91, 90, 87, 88, 91, 74, 92, 80, 92, 90, 78, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.880810338Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"":"downstairs","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"[1]R generation[0..12]: [58, 58, 56, 57, 58, 50, 59, 53, 59, 58, 52, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.880824166Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"":"downstairs","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.880838014Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"":"downstairs","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"[2]R flush_numbers[0..12]: [91, 90, 87, 88, 91, 74, 92, 80, 92, 90, 78, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.880851712Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"":"downstairs","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"[2]R generation[0..12]: [58, 58, 56, 57, 58, 50, 59, 53, 59, 58, 52, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.88086558Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"":"downstairs","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.880879668Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"":"downstairs","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.880892816Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"":"downstairs","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.880906194Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"":"downstairs","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"Next flush: 93","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.880920952Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"":"downstairs","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:32:55.880945708Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"":"downstairs","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.880977414Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.880991382Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"client":"0","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.8810049Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"client":"1","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.881026467Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"client":"2","":"downstairs","session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.881053003Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.881076449Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"f7f029a5-2ca5-4edc-85fa-88aba8f1b9cf is now active with session: 723aadc5-88b6-439e-b26a-dfacb8d65110","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.881100906Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.881122543Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427,"session_id":"723aadc5-88b6-439e-b26a-dfacb8d65110"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:32:55.881146299Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 541 len 16384 data: 5 8 8 7 02/30 Read block 328 len 40960 03/30 Read block 506 len 12288 04/30 Read block 72 len 8192 05/30 Read block 88 len 16384 06/30 Read block 586 len 8192 07/30 Read block 493 len 12288 08/30 Read block 495 len 4096 09/30 Write block 277 len 4096 data: 4 10/30 Write block 156 len 8192 data: 5 4 11/30 Flush 12/30 Write block 66 len 4096 data: 6 13/30 Read block 308 len 36864 14/30 Write block 144 len 16384 data: 6 6 6 7 15/30 Write block 155 len 24576 data: 5 6 5 6 7 8 16/30 Write block 63 len 28672 data: 7 9 6 7 7 6 5 17/30 Read block 403 len 28672 18/30 Flush 19/30 Write block 382 len 12288 data: 3 3 3 20/30 Read block 510 len 24576 21/30 Write block 300 len 36864 data: 5 5 5 3 4 4 4 4 5 22/30 Read block 357 len 20480 23/30 Read block 160 len 32768 24/30 Write block 243 len 20480 data: 8 8 9 7 8 25/30 Read block 359 len 12288 26/30 Write block 131 len 36864 data: 7 7 6 6 6 5 5 5 7 27/30 Read block 554 len 20480 28/30 Write block 513 len 8192 data: 6 6 29/30 Write block 68 len 12288 data: 7 6 7 30/30 Write block 508 len 28672 data: 4 4 3 3 5 7 7 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 3 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 3 Done Sent Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 1 4 Done 11 10 11 32 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:32:55.898888399Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1427} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 91 91 91 F F F 1 020-039 58 58 58 90 90 90 F F F 2 040-059 56 56 56 87 87 87 F F F 3 060-079 60 60 60 94 94 94 T T T 4 080-099 58 58 58 91 91 91 F F F 5 100-119 50 50 50 74 74 74 F F F 6 120-139 59 59 59 92 92 92 T T T 7 140-159 60 60 60 94 94 94 F F F 8 160-179 60 60 60 94 94 94 F F F 9 180-199 58 58 58 90 90 90 F F F 10 200-219 52 52 52 78 78 78 F F F 11 220-239 50 50 50 73 73 73 F F F 12 240-259 56 56 56 87 87 87 T T T 13 260-279 60 60 60 93 93 93 F F F 14 280-299 58 58 58 89 89 89 F F F 15 300-319 50 50 50 74 74 74 T T T 16 320-339 58 58 58 89 89 89 F F F 17 340-359 58 58 58 89 89 89 F F F 18 360-379 55 55 55 82 82 82 F F F 19 380-399 47 47 47 67 67 67 T T T 20 400-419 59 59 59 92 92 92 F F F 21 420-439 59 59 59 92 92 92 F F F 22 440-459 58 58 58 89 89 89 F F F 23 460-479 52 52 52 78 78 78 F F F 24 480-499 54 54 54 81 81 81 F F F 25 500-519 56 56 56 87 87 87 T T T 26 520-539 54 54 54 81 81 81 F F F 27 540-559 60 60 60 93 93 93 F F F 28 560-579 58 58 58 90 90 90 F F F 29 580-599 58 58 58 90 90 90 F F F Max gen: 60, Max flush: 94 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 91 91 91 F F F 1 020-039 58 58 58 90 90 90 F F F 2 040-059 56 56 56 87 87 87 F F F 3 060-079 61 61 61 95 95 95 F F F 4 080-099 58 58 58 91 91 91 F F F 5 100-119 50 50 50 74 74 74 F F F 6 120-139 61 61 61 95 95 95 F F F 7 140-159 60 60 60 94 94 94 F F F 8 160-179 60 60 60 94 94 94 F F F 9 180-199 58 58 58 90 90 90 F F F 10 200-219 52 52 52 78 78 78 F F F 11 220-239 50 50 50 73 73 73 F F F 12 240-259 61 61 61 95 95 95 F F F 13 260-279 60 60 60 93 93 93 F F F 14 280-299 58 58 58 89 89 89 F F F 15 300-319 61 61 61 95 95 95 F F F 16 320-339 58 58 58 89 89 89 F F F 17 340-359 58 58 58 89 89 89 F F F 18 360-379 55 55 55 82 82 82 F F F 19 380-399 61 61 61 95 95 95 F F F 20 400-419 59 59 59 92 92 92 F F F 21 420-439 59 59 59 92 92 92 F F F 22 440-459 58 58 58 89 89 89 F F F 23 460-479 52 52 52 78 78 78 F F F 24 480-499 54 54 54 81 81 81 F F F 25 500-519 61 61 61 95 95 95 F F F 26 520-539 54 54 54 81 81 81 F F F 27 540-559 60 60 60 93 93 93 F F F 28 560-579 58 58 58 90 90 90 F F F 29 580-599 58 58 58 90 90 90 F F F Max gen: 61, Max flush: 95 Begin loop 31 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.200526829Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.244944204Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.245868559Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.245907683Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.245923141Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.245938549Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435} {"msg":"Crucible 92e8de34-5e45-4077-aa8b-ed48d421e060 has session id: 685a016c-55d9-4211-aba0-022f1bdbe39d","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.246419659Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"Upstairs opts: Upstairs UUID: 92e8de34-5e45-4077-aa8b-ed48d421e060, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.246454514Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"Crucible stats registered with UUID: 92e8de34-5e45-4077-aa8b-ed48d421e060","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.246470491Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.276625628Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.276657244Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435} {"msg":"92e8de34-5e45-4077-aa8b-ed48d421e060 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.276689259Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.276711466Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"client":"0","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.276740381Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"client":"0","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.276758859Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"client":"1","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.276776626Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"client":"1","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.276793733Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"client":"2","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.276811461Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"client":"2","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.276828619Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"":"io task","client":"2","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.276852265Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"":"io task","client":"0","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.27688454Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"":"io task","client":"1","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"ds_connection connected from Ok(127.0.0.1:47291)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.276918985Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"":"io task","client":"2","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"ds_connection connected from Ok(127.0.0.1:45618)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.276950361Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"":"io task","client":"0","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"ds_connection connected from Ok(127.0.0.1:51176)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.277006102Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"":"io task","client":"1","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.27777891Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"client":"1","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.277809575Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"client":"1","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.27791671Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"client":"0","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.277938357Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"client":"0","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.278140057Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"client":"2","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.278176242Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"client":"2","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.278429055Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"client":"1","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.278616797Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"client":"0","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.27880182Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"client":"2","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.27901179Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"client":"1","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.279034796Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"client":"1","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.279049654Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.279164127Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"client":"0","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.279185994Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"client":"0","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.279199772Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.279311186Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"client":"2","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.279331893Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"client":"2","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"[0]R flush_numbers[0..12]: [91, 90, 87, 95, 91, 74, 95, 94, 94, 90, 78, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.279346261Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"":"downstairs","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"[0]R generation[0..12]: [58, 58, 56, 61, 58, 50, 61, 60, 60, 58, 52, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.279361149Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"":"downstairs","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.279378776Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"":"downstairs","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"[1]R flush_numbers[0..12]: [91, 90, 87, 95, 91, 74, 95, 94, 94, 90, 78, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.279392994Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"":"downstairs","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"[1]R generation[0..12]: [58, 58, 56, 61, 58, 50, 61, 60, 60, 58, 52, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.279406762Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"":"downstairs","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.27942095Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"":"downstairs","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"[2]R flush_numbers[0..12]: [91, 90, 87, 95, 91, 74, 95, 94, 94, 90, 78, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.279434878Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"":"downstairs","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"[2]R generation[0..12]: [58, 58, 56, 61, 58, 50, 61, 60, 60, 58, 52, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.279449306Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"":"downstairs","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.279463274Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"":"downstairs","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.279476442Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"":"downstairs","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.27948981Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"":"downstairs","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"Next flush: 96","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:33:00.279502838Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"":"downstairs","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.279520745Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"":"downstairs","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.279534173Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.279546222Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"client":"0","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.27955968Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"client":"1","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.279572808Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"client":"2","":"downstairs","session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.279588305Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.279600854Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"92e8de34-5e45-4077-aa8b-ed48d421e060 is now active with session: 685a016c-55d9-4211-aba0-022f1bdbe39d","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.279613722Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.27962591Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435,"session_id":"685a016c-55d9-4211-aba0-022f1bdbe39d"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:00.279638638Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 15 len 8192 data: 6 6 02/30 Write block 579 len 20480 data: 7 7 6 6 6 03/30 Write block 555 len 4096 data: 7 04/30 Read block 426 len 8192 05/30 Write block 56 len 16384 data: 5 7 8 7 06/30 Flush 07/30 Flush 08/30 Write block 209 len 28672 data: 4 4 5 5 7 8 5 09/30 Read block 357 len 16384 10/30 Read block 238 len 20480 11/30 Write block 83 len 40960 data: 6 5 9 6 4 5 7 7 6 6 12/30 Read block 378 len 36864 13/30 Write block 589 len 40960 data: 9 7 7 6 6 6 5 4 2 2 14/30 Read block 79 len 12288 15/30 Read block 134 len 20480 16/30 Read block 55 len 36864 17/30 Read block 516 len 24576 18/30 Write block 103 len 32768 data: 7 6 6 8 8 7 6 2 19/30 Read block 39 len 36864 20/30 Read block 191 len 20480 21/30 Read block 249 len 12288 22/30 Write block 9 len 16384 data: 4 4 4 4 23/30 Write block 51 len 40960 data: 7 8 6 6 4 6 8 9 8 6 24/30 Read block 366 len 36864 25/30 Read block 160 len 8192 26/30 Write block 469 len 36864 data: 6 6 6 6 6 4 6 7 6 27/30 Write block 428 len 36864 data: 6 5 4 4 5 3 3 6 6 28/30 Write block 398 len 24576 data: 8 8 8 6 4 4 29/30 Write block 102 len 36864 data: 6 8 7 7 9 9 8 7 3 30/30 Read block 498 len 20480 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 7 Done Done Done false 1008 Acked Read 4 Done Done Done false 1009 Acked Read 5 Done Done Done false 1010 Acked Write 10 Done Done Done false 1011 Acked Read 9 Done Done Done false 1012 Acked Write 10 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 9 Done Done Done false 1029 Acked Read 5 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 22 22 23 67 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1006 1006 1006 Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:33:00.299369237Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1435} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 96 96 96 T T T 1 020-039 58 58 58 90 90 90 F F F 2 040-059 62 62 62 96 96 96 T T T 3 060-079 61 61 61 95 95 95 T T T 4 080-099 58 58 58 91 91 91 T T T 5 100-119 50 50 50 74 74 74 T T T 6 120-139 61 61 61 95 95 95 F F F 7 140-159 60 60 60 94 94 94 F F F 8 160-179 60 60 60 94 94 94 F F F 9 180-199 58 58 58 90 90 90 F F F 10 200-219 52 52 52 78 78 78 T T T 11 220-239 50 50 50 73 73 73 F F F 12 240-259 61 61 61 95 95 95 F F F 13 260-279 60 60 60 93 93 93 F F F 14 280-299 58 58 58 89 89 89 F F F 15 300-319 61 61 61 95 95 95 F F F 16 320-339 58 58 58 89 89 89 F F F 17 340-359 58 58 58 89 89 89 F F F 18 360-379 55 55 55 82 82 82 F F F 19 380-399 61 61 61 95 95 95 T T T 20 400-419 59 59 59 92 92 92 T T T 21 420-439 59 59 59 92 92 92 T T T 22 440-459 58 58 58 89 89 89 F F F 23 460-479 52 52 52 78 78 78 T T T 24 480-499 54 54 54 81 81 81 F F F 25 500-519 61 61 61 95 95 95 F F F 26 520-539 54 54 54 81 81 81 F F F 27 540-559 62 62 62 96 96 96 F F F 28 560-579 62 62 62 96 96 96 F F F 29 580-599 62 62 62 96 96 96 T T T Max gen: 62, Max flush: 96 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 97 97 97 F F F 1 020-039 58 58 58 90 90 90 F F F 2 040-059 63 63 63 97 97 97 F F F 3 060-079 63 63 63 97 97 97 F F F 4 080-099 63 63 63 97 97 97 F F F 5 100-119 63 63 63 97 97 97 F F F 6 120-139 61 61 61 95 95 95 F F F 7 140-159 60 60 60 94 94 94 F F F 8 160-179 60 60 60 94 94 94 F F F 9 180-199 58 58 58 90 90 90 F F F 10 200-219 63 63 63 97 97 97 F F F 11 220-239 50 50 50 73 73 73 F F F 12 240-259 61 61 61 95 95 95 F F F 13 260-279 60 60 60 93 93 93 F F F 14 280-299 58 58 58 89 89 89 F F F 15 300-319 61 61 61 95 95 95 F F F 16 320-339 58 58 58 89 89 89 F F F 17 340-359 58 58 58 89 89 89 F F F 18 360-379 55 55 55 82 82 82 F F F 19 380-399 63 63 63 97 97 97 F F F 20 400-419 63 63 63 97 97 97 F F F 21 420-439 63 63 63 97 97 97 F F F 22 440-459 58 58 58 89 89 89 F F F 23 460-479 63 63 63 97 97 97 F F F 24 480-499 54 54 54 81 81 81 F F F 25 500-519 61 61 61 95 95 95 F F F 26 520-539 54 54 54 81 81 81 F F F 27 540-559 62 62 62 96 96 96 F F F 28 560-579 62 62 62 96 96 96 F F F 29 580-599 63 63 63 97 97 97 F F F Max gen: 63, Max flush: 97 Begin loop 32 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.596097824Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.64029834Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.640889994Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.640926028Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.640940566Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.640955144Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443} {"msg":"Crucible 9d36f8bb-800f-476a-8e24-bd29b96705e4 has session id: 596efe5e-6d7a-4e72-adf1-60b18e6675ee","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.641426775Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"Upstairs opts: Upstairs UUID: 9d36f8bb-800f-476a-8e24-bd29b96705e4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.641475038Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"Crucible stats registered with UUID: 9d36f8bb-800f-476a-8e24-bd29b96705e4","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.641494376Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.672290019Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.672327223Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443} {"msg":"9d36f8bb-800f-476a-8e24-bd29b96705e4 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.672356429Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.672377726Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"client":"0","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.672427139Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"client":"0","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.672466073Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"client":"1","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.672502748Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"client":"1","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.672538193Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"client":"2","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.672573887Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"client":"2","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.67262559Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"":"io task","client":"2","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.672661485Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"":"io task","client":"0","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.67269203Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"":"io task","client":"1","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:36366)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.672769429Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"":"io task","client":"2","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:53700)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.672810083Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"":"io task","client":"0","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:47035)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.672872714Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"":"io task","client":"1","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.673469297Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"client":"2","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.673497823Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"client":"2","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.6735136Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"client":"0","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.673528058Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"client":"0","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.673541796Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"client":"1","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.673648781Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"client":"1","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.674209739Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"client":"1","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.674349298Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"client":"0","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.674390162Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"client":"2","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.674772966Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"client":"1","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.674799912Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"client":"1","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.67481484Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.674908277Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"client":"0","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.674928234Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"client":"0","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.674941792Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.674995774Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"client":"2","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.675015211Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"client":"2","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"[0]R flush_numbers[0..12]: [97, 90, 97, 97, 97, 97, 95, 94, 94, 90, 97, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.675029539Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"":"downstairs","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"[0]R generation[0..12]: [63, 58, 63, 63, 63, 63, 61, 60, 60, 58, 63, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.675043627Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"":"downstairs","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.675057965Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"":"downstairs","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"[1]R flush_numbers[0..12]: [97, 90, 97, 97, 97, 97, 95, 94, 94, 90, 97, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.675075152Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"":"downstairs","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"[1]R generation[0..12]: [63, 58, 63, 63, 63, 63, 61, 60, 60, 58, 63, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.67508932Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"":"downstairs","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.675103478Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"":"downstairs","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"[2]R flush_numbers[0..12]: [97, 90, 97, 97, 97, 97, 95, 94, 94, 90, 97, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.675117346Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"":"downstairs","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"[2]R generation[0..12]: [63, 58, 63, 63, 63, 63, 61, 60, 60, 58, 63, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.675130894Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"":"downstairs","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.675144922Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"":"downstairs","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.67515807Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"":"downstairs","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.675171568Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"":"downstairs","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"Next flush: 98","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.675184596Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"":"downstairs","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"All extents match","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2025-01-03T08:33:04.675205633Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"":"downstairs","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.675219181Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.675231469Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"client":"0","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.675245037Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"client":"1","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.675258446Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"client":"2","":"downstairs","session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.675273793Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.675286541Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"9d36f8bb-800f-476a-8e24-bd29b96705e4 is now active with session: 596efe5e-6d7a-4e72-adf1-60b18e6675ee","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.675299589Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.675311808Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443,"session_id":"596efe5e-6d7a-4e72-adf1-60b18e6675ee"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:04.675324206Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 576 len 20480 data: 3 4 6 8 8 02/30 Write block 520 len 32768 data: 5 7 7 7 7 10 11 10 03/30 Write block 262 len 12288 data: 6 6 5 04/30 Read block 220 len 40960 05/30 Read block 10 len 8192 06/30 Read block 358 len 4096 07/30 Write block 584 len 4096 data: 8 08/30 Read block 374 len 4096 09/30 Write block 547 len 36864 data: 7 8 7 9 10 11 9 7 8 10/30 Read block 426 len 16384 11/30 Read block 236 len 40960 12/30 Read block 530 len 16384 13/30 Write block 262 len 28672 data: 7 7 6 4 4 3 4 14/30 Read block 286 len 20480 15/30 Read block 105 len 24576 16/30 Read block 146 len 40960 17/30 Read block 294 len 40960 18/30 Flush 19/30 Write block 132 len 8192 data: 8 7 20/30 Read block 359 len 36864 21/30 Write block 85 len 24576 data: 10 7 5 6 8 8 22/30 Write block 533 len 4096 data: 7 23/30 Read block 224 len 8192 24/30 Write block 84 len 24576 data: 6 11 8 6 7 9 25/30 Write block 312 len 28672 data: 5 5 5 6 5 5 5 26/30 Read block 85 len 24576 27/30 Read block 120 len 28672 28/30 Write block 50 len 12288 data: 8 8 9 29/30 Write block 66 len 12288 data: 8 8 8 30/30 Read block 48 len 24576 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 2 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Read 6 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:33:04.694085026Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1443} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 97 97 97 F F F 1 020-039 58 58 58 90 90 90 F F F 2 040-059 63 63 63 97 97 97 T T T 3 060-079 63 63 63 97 97 97 T T T 4 080-099 63 63 63 97 97 97 T T T 5 100-119 63 63 63 97 97 97 F F F 6 120-139 61 61 61 95 95 95 T T T 7 140-159 60 60 60 94 94 94 F F F 8 160-179 60 60 60 94 94 94 F F F 9 180-199 58 58 58 90 90 90 F F F 10 200-219 63 63 63 97 97 97 F F F 11 220-239 50 50 50 73 73 73 F F F 12 240-259 61 61 61 95 95 95 F F F 13 260-279 64 64 64 98 98 98 F F F 14 280-299 58 58 58 89 89 89 F F F 15 300-319 61 61 61 95 95 95 T T T 16 320-339 58 58 58 89 89 89 F F F 17 340-359 58 58 58 89 89 89 F F F 18 360-379 55 55 55 82 82 82 F F F 19 380-399 63 63 63 97 97 97 F F F 20 400-419 63 63 63 97 97 97 F F F 21 420-439 63 63 63 97 97 97 F F F 22 440-459 58 58 58 89 89 89 F F F 23 460-479 63 63 63 97 97 97 F F F 24 480-499 54 54 54 81 81 81 F F F 25 500-519 61 61 61 95 95 95 F F F 26 520-539 64 64 64 98 98 98 T T T 27 540-559 64 64 64 98 98 98 F F F 28 560-579 64 64 64 98 98 98 F F F 29 580-599 64 64 64 98 98 98 F F F Max gen: 64, Max flush: 98 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 97 97 97 F F F 1 020-039 58 58 58 90 90 90 F F F 2 040-059 65 65 65 99 99 99 F F F 3 060-079 65 65 65 99 99 99 F F F 4 080-099 65 65 65 99 99 99 F F F 5 100-119 63 63 63 97 97 97 F F F 6 120-139 65 65 65 99 99 99 F F F 7 140-159 60 60 60 94 94 94 F F F 8 160-179 60 60 60 94 94 94 F F F 9 180-199 58 58 58 90 90 90 F F F 10 200-219 63 63 63 97 97 97 F F F 11 220-239 50 50 50 73 73 73 F F F 12 240-259 61 61 61 95 95 95 F F F 13 260-279 64 64 64 98 98 98 F F F 14 280-299 58 58 58 89 89 89 F F F 15 300-319 65 65 65 99 99 99 F F F 16 320-339 58 58 58 89 89 89 F F F 17 340-359 58 58 58 89 89 89 F F F 18 360-379 55 55 55 82 82 82 F F F 19 380-399 63 63 63 97 97 97 F F F 20 400-419 63 63 63 97 97 97 F F F 21 420-439 63 63 63 97 97 97 F F F 22 440-459 58 58 58 89 89 89 F F F 23 460-479 63 63 63 97 97 97 F F F 24 480-499 54 54 54 81 81 81 F F F 25 500-519 61 61 61 95 95 95 F F F 26 520-539 65 65 65 99 99 99 F F F 27 540-559 64 64 64 98 98 98 F F F 28 560-579 64 64 64 98 98 98 F F F 29 580-599 64 64 64 98 98 98 F F F Max gen: 65, Max flush: 99 Begin loop 33 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:08.969289166Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:33:09.012426458Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.012875682Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.045103836Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.045758261Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.045798995Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.045816672Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.045835Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451} {"msg":"Crucible 78850afe-7531-424b-abbe-e8a064877275 has session id: 90f048ed-fb6d-444b-ae2d-911829101c2c","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.046276375Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"Upstairs opts: Upstairs UUID: 78850afe-7531-424b-abbe-e8a064877275, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.046332007Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"Crucible stats registered with UUID: 78850afe-7531-424b-abbe-e8a064877275","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.046362543Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.077436255Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.07747135Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451} {"msg":"78850afe-7531-424b-abbe-e8a064877275 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.077508765Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.077526922Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"client":"0","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.077553968Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"client":"0","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.077571436Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"client":"1","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.077593312Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"client":"1","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.07761071Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"client":"2","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.077628198Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"client":"2","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.077645375Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"":"io task","client":"2","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.077662852Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"":"io task","client":"0","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.07768008Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"":"io task","client":"1","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"ds_connection connected from Ok(127.0.0.1:43265)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.077762468Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"":"io task","client":"2","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"ds_connection connected from Ok(127.0.0.1:45013)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.078055095Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"":"io task","client":"0","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"ds_connection connected from Ok(127.0.0.1:63266)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.078102848Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"":"io task","client":"1","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.07850088Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"client":"2","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.078540764Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"client":"2","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.078600436Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"client":"1","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.078625222Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"client":"1","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.078883764Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"client":"0","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.078908471Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"client":"0","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.079215126Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"client":"1","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.079412687Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"client":"2","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.079567824Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"client":"0","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.079805249Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"client":"1","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.079834095Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"client":"1","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.079849893Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.080004391Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"client":"2","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.080031097Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"client":"2","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.080046324Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.080059063Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"client":"0","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.080072821Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"client":"0","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"[0]R flush_numbers[0..12]: [97, 90, 99, 99, 99, 97, 99, 94, 94, 90, 97, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.080097327Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"":"downstairs","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"[0]R generation[0..12]: [63, 58, 65, 65, 65, 63, 65, 60, 60, 58, 63, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.080112095Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"":"downstairs","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.080126343Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"":"downstairs","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"[1]R flush_numbers[0..12]: [97, 90, 99, 99, 99, 97, 99, 94, 94, 90, 97, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.080140211Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"":"downstairs","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"[1]R generation[0..12]: [63, 58, 65, 65, 65, 63, 65, 60, 60, 58, 63, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.080157388Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"":"downstairs","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.080172066Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"":"downstairs","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"[2]R flush_numbers[0..12]: [97, 90, 99, 99, 99, 97, 99, 94, 94, 90, 97, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.080186074Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"":"downstairs","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"[2]R generation[0..12]: [63, 58, 65, 65, 65, 63, 65, 60, 60, 58, 63, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.080200022Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"":"downstairs","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.08021416Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"":"downstairs","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.080227808Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"":"downstairs","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.080241626Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"":"downstairs","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"Next flush: 100","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.080254994Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"":"downstairs","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.080268112Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"":"downstairs","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:33:09.08028156Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.080303797Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"client":"0","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.080317155Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"client":"1","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.080330493Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"client":"2","":"downstairs","session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.080343891Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.080356359Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"78850afe-7531-424b-abbe-e8a064877275 is now active with session: 90f048ed-fb6d-444b-ae2d-911829101c2c","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.080371917Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.080384635Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451,"session_id":"90f048ed-fb6d-444b-ae2d-911829101c2c"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:09.080397243Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 217 len 16384 data: 5 5 5 5 02/30 Write block 69 len 32768 data: 7 8 6 5 5 5 7 7 03/30 Read block 154 len 40960 04/30 Write block 237 len 4096 data: 6 05/30 Read block 84 len 8192 06/30 Write block 378 len 12288 data: 5 5 5 07/30 Read block 303 len 12288 08/30 Write block 304 len 4096 data: 5 09/30 Read block 49 len 24576 10/30 Read block 449 len 16384 11/30 Write block 387 len 28672 data: 4 4 6 6 9 10 9 12/30 Write block 131 len 20480 data: 8 9 8 7 7 13/30 Read block 565 len 36864 14/30 Write block 457 len 8192 data: 6 7 15/30 Read block 258 len 20480 16/30 Read block 12 len 4096 17/30 Read block 37 len 8192 18/30 Read block 370 len 24576 19/30 Write block 477 len 32768 data: 7 4 5 6 5 5 4 3 20/30 Flush 21/30 Write block 499 len 20480 data: 8 8 7 6 5 22/30 Read block 111 len 40960 23/30 Write block 145 len 24576 data: 7 7 8 5 6 5 24/30 Write block 301 len 12288 data: 6 6 4 25/30 Flush 26/30 Write block 223 len 24576 data: 7 6 3 3 4 5 27/30 Read block 156 len 16384 28/30 Read block 262 len 32768 29/30 Write block 237 len 16384 data: 7 6 8 8 30/30 Write block 9 len 12288 data: 5 5 5 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 6 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:33:09.098889083Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1451} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 97 97 97 T T T 1 020-039 58 58 58 90 90 90 F F F 2 040-059 65 65 65 99 99 99 F F F 3 060-079 66 66 66 100 100 100 F F F 4 080-099 65 65 65 99 99 99 F F F 5 100-119 63 63 63 97 97 97 F F F 6 120-139 66 66 66 100 100 100 F F F 7 140-159 66 66 66 101 101 101 F F F 8 160-179 60 60 60 94 94 94 F F F 9 180-199 58 58 58 90 90 90 F F F 10 200-219 66 66 66 100 100 100 F F F 11 220-239 66 66 66 100 100 100 T T T 12 240-259 61 61 61 95 95 95 T T T 13 260-279 64 64 64 98 98 98 F F F 14 280-299 58 58 58 89 89 89 F F F 15 300-319 66 66 66 101 101 101 F F F 16 320-339 58 58 58 89 89 89 F F F 17 340-359 58 58 58 89 89 89 F F F 18 360-379 66 66 66 100 100 100 F F F 19 380-399 66 66 66 100 100 100 F F F 20 400-419 63 63 63 97 97 97 F F F 21 420-439 63 63 63 97 97 97 F F F 22 440-459 66 66 66 100 100 100 F F F 23 460-479 66 66 66 100 100 100 F F F 24 480-499 66 66 66 101 101 101 F F F 25 500-519 66 66 66 101 101 101 F F F 26 520-539 65 65 65 99 99 99 F F F 27 540-559 64 64 64 98 98 98 F F F 28 560-579 64 64 64 98 98 98 F F F 29 580-599 64 64 64 98 98 98 F F F Max gen: 66, Max flush: 101 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 67 67 67 102 102 102 F F F 1 020-039 58 58 58 90 90 90 F F F 2 040-059 65 65 65 99 99 99 F F F 3 060-079 66 66 66 100 100 100 F F F 4 080-099 65 65 65 99 99 99 F F F 5 100-119 63 63 63 97 97 97 F F F 6 120-139 66 66 66 100 100 100 F F F 7 140-159 66 66 66 101 101 101 F F F 8 160-179 60 60 60 94 94 94 F F F 9 180-199 58 58 58 90 90 90 F F F 10 200-219 66 66 66 100 100 100 F F F 11 220-239 67 67 67 102 102 102 F F F 12 240-259 67 67 67 102 102 102 F F F 13 260-279 64 64 64 98 98 98 F F F 14 280-299 58 58 58 89 89 89 F F F 15 300-319 66 66 66 101 101 101 F F F 16 320-339 58 58 58 89 89 89 F F F 17 340-359 58 58 58 89 89 89 F F F 18 360-379 66 66 66 100 100 100 F F F 19 380-399 66 66 66 100 100 100 F F F 20 400-419 63 63 63 97 97 97 F F F 21 420-439 63 63 63 97 97 97 F F F 22 440-459 66 66 66 100 100 100 F F F 23 460-479 66 66 66 100 100 100 F F F 24 480-499 66 66 66 101 101 101 F F F 25 500-519 66 66 66 101 101 101 F F F 26 520-539 65 65 65 99 99 99 F F F 27 540-559 64 64 64 98 98 98 F F F 28 560-579 64 64 64 98 98 98 F F F 29 580-599 64 64 64 98 98 98 F F F Max gen: 67, Max flush: 102 Begin loop 34 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.386597528Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.432213197Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.432821699Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.432857343Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.43287642Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.432891618Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459} {"msg":"Crucible 68d0ab02-2db1-4131-a7f1-f4e5631368df has session id: 38c1f8c6-a131-4806-b13a-8d37874170cc","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.433391535Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"Upstairs opts: Upstairs UUID: 68d0ab02-2db1-4131-a7f1-f4e5631368df, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.433443538Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"Crucible stats registered with UUID: 68d0ab02-2db1-4131-a7f1-f4e5631368df","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.433479592Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.463615992Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.463661715Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459} {"msg":"68d0ab02-2db1-4131-a7f1-f4e5631368df active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.4636956Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.463726306Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"client":"0","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.463774759Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"client":"0","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.463830271Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"client":"1","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.463866795Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"client":"1","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.463899591Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"client":"2","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.463932726Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"client":"2","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.463963241Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"":"io task","client":"2","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.463999326Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"":"io task","client":"0","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.464206616Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"":"io task","client":"1","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:44747)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.46424505Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"":"io task","client":"2","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:39630)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.464268217Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"":"io task","client":"1","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:57608)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.464287224Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"":"io task","client":"0","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.464819556Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"client":"1","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.464849352Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"client":"1","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.46486527Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"client":"0","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.464879638Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"client":"0","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.465015288Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"client":"2","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.465047853Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"client":"2","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.465453864Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"client":"0","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.465494558Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"client":"1","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.465567677Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"client":"2","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.466045517Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"client":"1","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.466077343Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"client":"1","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.46609272Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.46616516Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"client":"2","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.466185087Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"client":"2","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.466198955Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.466211393Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"client":"0","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.466225111Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"client":"0","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"[0]R flush_numbers[0..12]: [102, 90, 99, 100, 99, 97, 100, 101, 94, 90, 100, 102]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.466239159Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"":"downstairs","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"[0]R generation[0..12]: [67, 58, 65, 66, 65, 63, 66, 66, 60, 58, 66, 67]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.466253627Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"":"downstairs","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.466267905Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"":"downstairs","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"[1]R flush_numbers[0..12]: [102, 90, 99, 100, 99, 97, 100, 101, 94, 90, 100, 102]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.466281783Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"":"downstairs","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"[1]R generation[0..12]: [67, 58, 65, 66, 65, 63, 66, 66, 60, 58, 66, 67]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.466295461Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"":"downstairs","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.466310509Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"":"downstairs","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"[2]R flush_numbers[0..12]: [102, 90, 99, 100, 99, 97, 100, 101, 94, 90, 100, 102]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.466324436Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"":"downstairs","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"[2]R generation[0..12]: [67, 58, 65, 66, 65, 63, 66, 66, 60, 58, 66, 67]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.466338125Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"":"downstairs","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.466354932Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"":"downstairs","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.4663685Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"":"downstairs","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.466381988Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"":"downstairs","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"Wait for a query_work_queue command to finish before sending IO } {"msg":"Next flush: 103","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.466406155Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"":"downstairs","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.466419373Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"":"downstairs","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.466432981Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.466445119Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"client":"0","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.466458537Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"client":"1","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.466472035Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"client":"2","":"downstairs","session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.466485663Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.466498071Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"68d0ab02-2db1-4131-a7f1-f4e5631368df is now active with session: 38c1f8c6-a131-4806-b13a-8d37874170cc","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.466510909Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.466523178Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459,"session_id":"38c1f8c6-a131-4806-b13a-8d37874170cc"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:13.466535596Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 562 len 24576 data: 5 5 7 5 6 6 02/30 Read block 212 len 32768 03/30 Write block 486 len 8192 data: 2 2 04/30 Read block 228 len 20480 05/30 Flush 06/30 Write block 266 len 32768 data: 5 4 5 3 5 7 4 5 07/30 Read block 7 len 20480 08/30 Read block 466 len 8192 09/30 Write block 186 len 24576 data: 9 9 9 9 9 8 10/30 Write block 402 len 40960 data: 5 5 5 5 6 5 6 6 7 9 11/30 Write block 245 len 4096 data: 10 12/30 Write block 483 len 12288 data: 5 4 3 13/30 Flush 14/30 Write block 224 len 28672 data: 7 4 4 5 6 5 6 15/30 Read block 160 len 16384 16/30 Read block 240 len 16384 17/30 Read block 375 len 20480 18/30 Write block 167 len 4096 data: 7 19/30 Flush 20/30 Write block 237 len 8192 data: 8 7 21/30 Read block 468 len 28672 22/30 Write block 232 len 32768 data: 6 7 8 8 8 9 8 9 23/30 Read block 26 len 28672 24/30 Write block 72 len 12288 data: 6 6 6 25/30 Read block 260 len 8192 26/30 Write block 312 len 32768 data: 6 6 6 7 6 6 6 4 27/30 Read block 138 len 12288 28/30 Read block 421 len 28672 29/30 Read block 489 len 12288 30/30 Write block 74 len 36864 data: 7 8 8 7 7 7 8 8 6 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 2 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:33:13.485147278Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1459} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 67 67 67 102 102 102 F F F 1 020-039 58 58 58 90 90 90 F F F 2 040-059 65 65 65 99 99 99 F F F 3 060-079 66 66 66 100 100 100 T T T 4 080-099 65 65 65 99 99 99 T T T 5 100-119 63 63 63 97 97 97 F F F 6 120-139 66 66 66 100 100 100 F F F 7 140-159 66 66 66 101 101 101 F F F 8 160-179 68 68 68 105 105 105 F F F 9 180-199 68 68 68 104 104 104 F F F 10 200-219 66 66 66 100 100 100 F F F 11 220-239 68 68 68 105 105 105 T T T 12 240-259 68 68 68 104 104 104 F F F 13 260-279 68 68 68 104 104 104 F F F 14 280-299 58 58 58 89 89 89 F F F 15 300-319 66 66 66 101 101 101 T T T 16 320-339 58 58 58 89 89 89 F F F 17 340-359 58 58 58 89 89 89 F F F 18 360-379 66 66 66 100 100 100 F F F 19 380-399 66 66 66 100 100 100 F F F 20 400-419 68 68 68 104 104 104 F F F 21 420-439 63 63 63 97 97 97 F F F 22 440-459 66 66 66 100 100 100 F F F 23 460-479 66 66 66 100 100 100 F F F 24 480-499 68 68 68 104 104 104 F F F 25 500-519 66 66 66 101 101 101 F F F 26 520-539 65 65 65 99 99 99 F F F 27 540-559 64 64 64 98 98 98 F F F 28 560-579 68 68 68 103 103 103 F F F 29 580-599 64 64 64 98 98 98 F F F Max gen: 68, Max flush: 105 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 67 67 67 102 102 102 F F F 1 020-039 58 58 58 90 90 90 F F F 2 040-059 65 65 65 99 99 99 F F F 3 060-079 69 69 69 106 106 106 F F F 4 080-099 69 69 69 106 106 106 F F F 5 100-119 63 63 63 97 97 97 F F F 6 120-139 66 66 66 100 100 100 F F F 7 140-159 66 66 66 101 101 101 F F F 8 160-179 68 68 68 105 105 105 F F F 9 180-199 68 68 68 104 104 104 F F F 10 200-219 66 66 66 100 100 100 F F F 11 220-239 69 69 69 106 106 106 F F F 12 240-259 68 68 68 104 104 104 F F F 13 260-279 68 68 68 104 104 104 F F F 14 280-299 58 58 58 89 89 89 F F F 15 300-319 69 69 69 106 106 106 F F F 16 320-339 58 58 58 89 89 89 F F F 17 340-359 58 58 58 89 89 89 F F F 18 360-379 66 66 66 100 100 100 F F F 19 380-399 66 66 66 100 100 100 F F F 20 400-419 68 68 68 104 104 104 F F F 21 420-439 63 63 63 97 97 97 F F F 22 440-459 66 66 66 100 100 100 F F F 23 460-479 66 66 66 100 100 100 F F F 24 480-499 68 68 68 104 104 104 F F F 25 500-519 66 66 66 101 101 101 F F F 26 520-539 65 65 65 99 99 99 F F F 27 540-559 64 64 64 98 98 98 F F F 28 560-579 68 68 68 103 103 103 F F F 29 580-599 64 64 64 98 98 98 F F F Max gen: 69, Max flush: 106 Begin loop 35 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.744664319Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.788913438Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.789604847Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.789639592Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.78965419Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.789668887Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467} {"msg":"Crucible 44ed7bf3-6160-42df-83d6-65039817a2f9 has session id: 21e8ae60-cbbc-466b-920b-5b550066cf17","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.79013493Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"Upstairs opts: Upstairs UUID: 44ed7bf3-6160-42df-83d6-65039817a2f9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.790243244Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"Crucible stats registered with UUID: 44ed7bf3-6160-42df-83d6-65039817a2f9","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.790276639Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.820599881Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.820631937Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467} {"msg":"44ed7bf3-6160-42df-83d6-65039817a2f9 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.820667991Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.820686049Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"client":"0","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.820711925Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"client":"0","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.820729772Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"client":"1","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.820752069Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"client":"1","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.820783884Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"client":"2","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.820817779Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"client":"2","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.820853354Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"":"io task","client":"2","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.820889259Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"":"io task","client":"0","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.820925574Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"":"io task","client":"1","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"ds_connection connected from Ok(127.0.0.1:39416)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.82095363Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"":"io task","client":"2","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"ds_connection connected from Ok(127.0.0.1:43559)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.820989814Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"":"io task","client":"0","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"ds_connection connected from Ok(127.0.0.1:53388)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.821217251Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"":"io task","client":"1","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.821724447Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"client":"0","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.821764851Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"client":"0","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.821797556Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"client":"2","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.821830112Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"client":"2","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.821860467Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"client":"1","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.821887013Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"client":"1","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.822441612Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"client":"0","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.822486336Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"client":"2","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.822540678Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"client":"1","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.823015269Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"client":"0","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.823043725Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"client":"0","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.823069431Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.823133411Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"client":"1","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.823156678Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"client":"1","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.823170856Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.823183104Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"client":"2","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.82320917Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"client":"2","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"[0]R flush_numbers[0..12]: [102, 90, 99, 106, 106, 97, 100, 101, 105, 104, 100, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.823225128Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"":"downstairs","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"[0]R generation[0..12]: [67, 58, 65, 69, 69, 63, 66, 66, 68, 68, 66, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.823239206Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"":"downstairs","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.823254774Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"":"downstairs","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"[1]R flush_numbers[0..12]: [102, 90, 99, 106, 106, 97, 100, 101, 105, 104, 100, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.823269092Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"":"downstairs","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"[1]R generation[0..12]: [67, 58, 65, 69, 69, 63, 66, 66, 68, 68, 66, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.823282739Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"":"downstairs","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.823296897Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"":"downstairs","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"[2]R flush_numbers[0..12]: [102, 90, 99, 106, 106, 97, 100, 101, 105, 104, 100, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.823310806Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"":"downstairs","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"[2]R generation[0..12]: [67, 58, 65, 69, 69, 63, 66, 66, 68, 68, 66, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.823324384Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"":"downstairs","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.823338291Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"":"downstairs","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.823351369Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"":"downstairs","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.823366967Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"":"downstairs","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"Next flush: 107","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.823380265Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"":"downstairs","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:33:17.823393463Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"":"downstairs","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.823466493Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.82348175Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"client":"0","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.823495288Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"client":"1","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.823508666Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"client":"2","":"downstairs","session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.823521995Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.823534603Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"44ed7bf3-6160-42df-83d6-65039817a2f9 is now active with session: 21e8ae60-cbbc-466b-920b-5b550066cf17","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.823547521Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.823559549Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467,"session_id":"21e8ae60-cbbc-466b-920b-5b550066cf17"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:17.823571747Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 9 len 32768 data: 6 6 6 5 6 6 7 7 02/30 Write block 488 len 20480 data: 2 2 4 5 5 03/30 Write block 529 len 36864 data: 9 7 7 7 8 6 5 5 5 04/30 Read block 207 len 20480 05/30 Read block 299 len 8192 06/30 Read block 366 len 36864 07/30 Read block 273 len 16384 08/30 Read block 77 len 16384 09/30 Read block 472 len 32768 10/30 Write block 76 len 16384 data: 9 8 8 8 11/30 Read block 24 len 4096 12/30 Write block 578 len 40960 data: 7 9 9 7 7 7 9 6 6 8 13/30 Read block 158 len 4096 14/30 Read block 89 len 20480 15/30 Read block 405 len 24576 16/30 Write block 390 len 20480 data: 7 10 11 10 9 17/30 Read block 224 len 32768 18/30 Read block 92 len 40960 19/30 Flush 20/30 Write block 458 len 16384 data: 8 7 6 5 21/30 Write block 582 len 24576 data: 8 8 10 7 7 9 22/30 Write block 586 len 40960 data: 8 10 10 10 8 8 7 7 7 6 23/30 Read block 319 len 24576 24/30 Write block 196 len 4096 data: 5 25/30 Write block 161 len 32768 data: 10 11 9 9 9 8 8 5 26/30 Read block 323 len 32768 27/30 Read block 540 len 28672 28/30 Read block 320 len 20480 29/30 Write block 215 len 40960 data: 6 5 6 6 6 6 7 6 8 8 30/30 Write block 519 len 28672 data: 5 6 8 8 8 8 11 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 4 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:33:17.843547691Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1467} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 107 107 107 F F F 1 020-039 58 58 58 90 90 90 F F F 2 040-059 65 65 65 99 99 99 F F F 3 060-079 70 70 70 107 107 107 F F F 4 080-099 69 69 69 106 106 106 F F F 5 100-119 63 63 63 97 97 97 F F F 6 120-139 66 66 66 100 100 100 F F F 7 140-159 66 66 66 101 101 101 F F F 8 160-179 68 68 68 105 105 105 T T T 9 180-199 68 68 68 104 104 104 T T T 10 200-219 66 66 66 100 100 100 T T T 11 220-239 69 69 69 106 106 106 T T T 12 240-259 68 68 68 104 104 104 F F F 13 260-279 68 68 68 104 104 104 F F F 14 280-299 58 58 58 89 89 89 F F F 15 300-319 69 69 69 106 106 106 F F F 16 320-339 58 58 58 89 89 89 F F F 17 340-359 58 58 58 89 89 89 F F F 18 360-379 66 66 66 100 100 100 F F F 19 380-399 70 70 70 107 107 107 F F F 20 400-419 68 68 68 104 104 104 F F F 21 420-439 63 63 63 97 97 97 F F F 22 440-459 66 66 66 100 100 100 T T T 23 460-479 66 66 66 100 100 100 T T T 24 480-499 70 70 70 107 107 107 F F F 25 500-519 66 66 66 101 101 101 T T T 26 520-539 70 70 70 107 107 107 T T T 27 540-559 64 64 64 98 98 98 F F F 28 560-579 70 70 70 107 107 107 F F F 29 580-599 70 70 70 107 107 107 T T T Max gen: 70, Max flush: 107 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 107 107 107 F F F 1 020-039 58 58 58 90 90 90 F F F 2 040-059 65 65 65 99 99 99 F F F 3 060-079 70 70 70 107 107 107 F F F 4 080-099 69 69 69 106 106 106 F F F 5 100-119 63 63 63 97 97 97 F F F 6 120-139 66 66 66 100 100 100 F F F 7 140-159 66 66 66 101 101 101 F F F 8 160-179 71 71 71 108 108 108 F F F 9 180-199 71 71 71 108 108 108 F F F 10 200-219 71 71 71 108 108 108 F F F 11 220-239 71 71 71 108 108 108 F F F 12 240-259 68 68 68 104 104 104 F F F 13 260-279 68 68 68 104 104 104 F F F 14 280-299 58 58 58 89 89 89 F F F 15 300-319 69 69 69 106 106 106 F F F 16 320-339 58 58 58 89 89 89 F F F 17 340-359 58 58 58 89 89 89 F F F 18 360-379 66 66 66 100 100 100 F F F 19 380-399 70 70 70 107 107 107 F F F 20 400-419 68 68 68 104 104 104 F F F 21 420-439 63 63 63 97 97 97 F F F 22 440-459 71 71 71 108 108 108 F F F 23 460-479 71 71 71 108 108 108 F F F 24 480-499 70 70 70 107 107 107 F F F 25 500-519 71 71 71 108 108 108 F F F 26 520-539 71 71 71 108 108 108 F F F 27 540-559 64 64 64 98 98 98 F F F 28 560-579 70 70 70 107 107 107 F F F 29 580-599 71 71 71 108 108 108 F F F Max gen: 71, Max flush: 108 Begin loop 36 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.131072052Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.175206297Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.175855973Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.175892517Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.175908925Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.175924473Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475} {"msg":"Crucible 2df71cf4-db9b-47b9-9988-53363f4f6940 has session id: 5c70d650-6f20-4c65-88e7-513f707acead","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.176384955Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"Upstairs opts: Upstairs UUID: 2df71cf4-db9b-47b9-9988-53363f4f6940, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.176433458Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"Crucible stats registered with UUID: 2df71cf4-db9b-47b9-9988-53363f4f6940","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.176466593Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.206604483Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.206639118Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475} {"msg":"2df71cf4-db9b-47b9-9988-53363f4f6940 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.20669106Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.206712097Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"client":"0","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.206738303Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"client":"0","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.206756301Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"client":"1","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.206773738Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"client":"1","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.206791316Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"client":"2","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.206808673Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"client":"2","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.206825791Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"":"io task","client":"2","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.206849907Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"":"io task","client":"0","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.207123897Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"":"io task","client":"1","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"ds_connection connected from Ok(127.0.0.1:54275)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.20717209Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"":"io task","client":"2","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"ds_connection connected from Ok(127.0.0.1:39177)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.207209935Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"":"io task","client":"1","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"ds_connection connected from Ok(127.0.0.1:36757)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.2072422Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"":"io task","client":"0","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.207737267Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"client":"2","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.207765223Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"client":"2","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.207780471Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"client":"0","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.207799298Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"client":"0","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.207874358Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"client":"1","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.207908193Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"client":"1","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.208380074Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"client":"2","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.208411849Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"client":"0","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.208513154Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"client":"1","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.208972727Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"client":"0","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.209002183Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"client":"0","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.20901852Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.209031878Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"client":"1","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.209046056Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"client":"1","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.209059705Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.209071993Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"client":"2","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.20908921Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"client":"2","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"[0]R flush_numbers[0..12]: [107, 90, 99, 107, 106, 97, 100, 101, 108, 108, 108, 108]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.209103918Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"":"downstairs","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"[0]R generation[0..12]: [70, 58, 65, 70, 69, 63, 66, 66, 71, 71, 71, 71]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.209117826Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"":"downstairs","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.209132304Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"":"downstairs","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"[1]R flush_numbers[0..12]: [107, 90, 99, 107, 106, 97, 100, 101, 108, 108, 108, 108]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.209146342Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"":"downstairs","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"[1]R generation[0..12]: [70, 58, 65, 70, 69, 63, 66, 66, 71, 71, 71, 71]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.20916013Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"":"downstairs","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.209174208Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"":"downstairs","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"[2]R flush_numbers[0..12]: [107, 90, 99, 107, 106, 97, 100, 101, 108, 108, 108, 108]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.209188206Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"":"downstairs","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"[2]R generation[0..12]: [70, 58, 65, 70, 69, 63, 66, 66, 71, 71, 71, 71]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.209201934Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"":"downstairs","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.209216012Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"":"downstairs","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.20922934Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"":"downstairs","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.209242958Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"":"downstairs","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"Next flush: 109","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:33:22.209256066Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"":"downstairs","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.209281862Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"":"downstairs","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.20929547Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.209307728Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"client":"0","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.209321216Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"client":"1","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.209334354Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"client":"2","":"downstairs","session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.20936379Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.209376508Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"2df71cf4-db9b-47b9-9988-53363f4f6940 is now active with session: 5c70d650-6f20-4c65-88e7-513f707acead","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.209389496Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.209401855Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475,"session_id":"5c70d650-6f20-4c65-88e7-513f707acead"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:22.209414313Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 97 len 32768 data: 5 6 8 5 5 7 9 8 02/30 Write block 240 len 20480 data: 9 9 9 9 9 03/30 Write block 524 len 16384 data: 9 12 12 11 04/30 Read block 575 len 24576 05/30 Write block 489 len 8192 data: 3 5 06/30 Write block 82 len 8192 data: 7 7 07/30 Write block 245 len 20480 data: 11 8 9 8 6 08/30 Write block 382 len 24576 data: 4 4 4 3 3 5 09/30 Read block 288 len 16384 10/30 Write block 44 len 36864 data: 5 6 8 6 7 8 9 9 10 11/30 Read block 252 len 36864 12/30 Write block 116 len 32768 data: 5 6 9 11 11 10 9 10 13/30 Write block 82 len 8192 data: 8 8 14/30 Read block 113 len 28672 15/30 Read block 162 len 28672 16/30 Read block 248 len 28672 17/30 Write block 50 len 20480 data: 10 10 11 7 7 18/30 Read block 9 len 16384 19/30 Write block 234 len 28672 data: 9 9 9 10 9 10 10 20/30 Read block 308 len 4096 21/30 Write block 448 len 32768 data: 6 6 6 6 5 6 6 7 22/30 Read block 45 len 20480 23/30 Write block 226 len 4096 data: 5 24/30 Read block 374 len 40960 25/30 Read block 60 len 8192 26/30 Read block 209 len 16384 27/30 Write block 492 len 12288 data: 6 8 7 28/30 Write block 60 len 40960 data: 7 6 8 8 10 7 9 9 9 8 29/30 Read block 401 len 12288 30/30 Write block 39 len 20480 data: 6 5 5 7 6 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 5 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Write 2 Done Done Done false 1005 Acked Write 2 Done Done Done false 1006 Acked Write 5 Done Done Done false 1007 Acked Write 6 Done Done Done false 1008 Acked Read 4 Done Done Done false 1009 Acked Write 9 Done Done Done false 1010 Acked Read 9 Done Done Done false 1011 Acked Write 8 Done Done Done false 1012 Acked Write 2 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:33:22.228213198Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1475} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 107 107 107 F F F 1 020-039 58 58 58 90 90 90 T T T 2 040-059 65 65 65 99 99 99 T T T 3 060-079 70 70 70 107 107 107 T T T 4 080-099 69 69 69 106 106 106 T T T 5 100-119 63 63 63 97 97 97 T T T 6 120-139 66 66 66 100 100 100 T T T 7 140-159 66 66 66 101 101 101 F F F 8 160-179 71 71 71 108 108 108 F F F 9 180-199 71 71 71 108 108 108 F F F 10 200-219 71 71 71 108 108 108 F F F 11 220-239 71 71 71 108 108 108 T T T 12 240-259 68 68 68 104 104 104 T T T 13 260-279 68 68 68 104 104 104 F F F 14 280-299 58 58 58 89 89 89 F F F 15 300-319 69 69 69 106 106 106 F F F 16 320-339 58 58 58 89 89 89 F F F 17 340-359 58 58 58 89 89 89 F F F 18 360-379 66 66 66 100 100 100 F F F 19 380-399 70 70 70 107 107 107 T T T 20 400-419 68 68 68 104 104 104 F F F 21 420-439 63 63 63 97 97 97 F F F 22 440-459 71 71 71 108 108 108 T T T 23 460-479 71 71 71 108 108 108 F F F 24 480-499 70 70 70 107 107 107 T T T 25 500-519 71 71 71 108 108 108 F F F 26 520-539 71 71 71 108 108 108 T T T 27 540-559 64 64 64 98 98 98 F F F 28 560-579 70 70 70 107 107 107 F F F 29 580-599 71 71 71 108 108 108 F F F Max gen: 71, Max flush: 108 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 107 107 107 F F F 1 020-039 72 72 72 109 109 109 F F F 2 040-059 72 72 72 109 109 109 F F F 3 060-079 72 72 72 109 109 109 F F F 4 080-099 72 72 72 109 109 109 F F F 5 100-119 72 72 72 109 109 109 F F F 6 120-139 72 72 72 109 109 109 F F F 7 140-159 66 66 66 101 101 101 F F F 8 160-179 71 71 71 108 108 108 F F F 9 180-199 71 71 71 108 108 108 F F F 10 200-219 71 71 71 108 108 108 F F F 11 220-239 72 72 72 109 109 109 F F F 12 240-259 72 72 72 109 109 109 F F F 13 260-279 68 68 68 104 104 104 F F F 14 280-299 58 58 58 89 89 89 F F F 15 300-319 69 69 69 106 106 106 F F F 16 320-339 58 58 58 89 89 89 F F F 17 340-359 58 58 58 89 89 89 F F F 18 360-379 66 66 66 100 100 100 F F F 19 380-399 72 72 72 109 109 109 F F F 20 400-419 68 68 68 104 104 104 F F F 21 420-439 63 63 63 97 97 97 F F F 22 440-459 72 72 72 109 109 109 F F F 23 460-479 71 71 71 108 108 108 F F F 24 480-499 72 72 72 109 109 109 F F F 25 500-519 71 71 71 108 108 108 F F F 26 520-539 72 72 72 109 109 109 F F F 27 540-559 64 64 64 98 98 98 F F F 28 560-579 70 70 70 107 107 107 F F F 29 580-599 71 71 71 108 108 108 F F F Max gen: 72, Max flush: 109 Begin loop 37 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.52887963Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.572507989Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.573096033Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.573146486Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.573161844Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.573176682Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483} {"msg":"Crucible 33f44b8c-dd1d-4cdc-957b-a990d036e27a has session id: 356f92ce-fe64-445b-9c38-521d9b234d55","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.573720182Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"Upstairs opts: Upstairs UUID: 33f44b8c-dd1d-4cdc-957b-a990d036e27a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.573770235Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"Crucible stats registered with UUID: 33f44b8c-dd1d-4cdc-957b-a990d036e27a","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.573795641Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.603880188Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.603919033Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483} {"msg":"33f44b8c-dd1d-4cdc-957b-a990d036e27a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.603945339Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.603969715Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"client":"0","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.60400748Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"client":"0","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.604039805Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"client":"1","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.604070311Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"client":"1","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.604097367Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"client":"2","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.604125133Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"client":"2","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.604152969Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"":"io task","client":"0","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.604180755Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"":"io task","client":"2","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.60420989Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"":"io task","client":"1","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"ds_connection connected from Ok(127.0.0.1:42018)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.604240306Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"":"io task","client":"0","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"ds_connection connected from Ok(127.0.0.1:32871)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.604259613Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"":"io task","client":"2","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"ds_connection connected from Ok(127.0.0.1:59504)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.6042815Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"":"io task","client":"1","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.605018322Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"client":"2","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.605046578Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"client":"2","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.605061606Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"client":"0","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.605075314Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"client":"0","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.605178839Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"client":"1","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.605209014Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"client":"1","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.605641551Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"client":"0","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.605723859Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"client":"1","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.605802978Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"client":"2","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.606213168Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"client":"0","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.606240504Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"client":"0","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.606255312Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.606327141Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"client":"1","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.606361716Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"client":"1","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.606388032Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.6064018Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"client":"2","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.606415638Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"client":"2","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"[0]R flush_numbers[0..12]: [107, 109, 109, 109, 109, 109, 109, 101, 108, 108, 108, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.606429706Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"":"downstairs","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"[0]R generation[0..12]: [70, 72, 72, 72, 72, 72, 72, 66, 71, 71, 71, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.606443634Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"":"downstairs","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.606457682Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"":"downstairs","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"[1]R flush_numbers[0..12]: [107, 109, 109, 109, 109, 109, 109, 101, 108, 108, 108, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.60647162Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"":"downstairs","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"[1]R generation[0..12]: [70, 72, 72, 72, 72, 72, 72, 66, 71, 71, 71, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.606485398Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"":"downstairs","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.606499426Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"":"downstairs","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"[2]R flush_numbers[0..12]: [107, 109, 109, 109, 109, 109, 109, 101, 108, 108, 108, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.606513284Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"":"downstairs","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"[2]R generation[0..12]: [70, 72, 72, 72, 72, 72, 72, 66, 71, 71, 71, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.606526992Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"":"downstairs","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.60654297Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"":"downstairs","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"Max found gen is 73","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.606567726ZWait for a query_work_queue command to finish before sending IO ","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"":"downstairs","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"Generation requested: 74 >= found:73","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.606603291Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"":"downstairs","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"Next flush: 110","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.606623368Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"":"downstairs","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.606636956Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"":"downstairs","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.606650624Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.606662922Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"client":"0","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.60667649Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"client":"1","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.606689798Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"client":"2","":"downstairs","session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.606703066Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.606715614Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"33f44b8c-dd1d-4cdc-957b-a990d036e27a is now active with session: 356f92ce-fe64-445b-9c38-521d9b234d55","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.606728643Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.606740731Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483,"session_id":"356f92ce-fe64-445b-9c38-521d9b234d55"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:26.606753199Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 317 len 36864 data: 7 7 5 4 4 4 4 5 5 02/30 Write block 252 len 20480 data: 10 9 9 7 8 03/30 Read block 217 len 16384 04/30 Read block 357 len 24576 05/30 Read block 472 len 20480 06/30 Write block 318 len 24576 data: 8 6 5 5 5 5 07/30 Write block 111 len 28672 data: 2 3 4 5 5 6 7 08/30 Write block 13 len 36864 data: 7 7 8 8 7 8 10 8 9 09/30 Read block 574 len 32768 10/30 Flush 11/30 Write block 476 len 12288 data: 8 8 5 12/30 Read block 575 len 8192 13/30 Read block 2 len 28672 14/30 Write block 85 len 24576 data: 12 9 7 8 10 9 15/30 Read block 309 len 20480 16/30 Read block 497 len 4096 17/30 Read block 468 len 4096 18/30 Write block 57 len 40960 data: 9 10 9 8 7 9 9 11 8 10 19/30 Write block 24 len 32768 data: 7 5 4 4 4 5 6 5 20/30 Read block 475 len 8192 21/30 Write block 546 len 40960 data: 7 8 9 8 10 11 12 10 8 9 22/30 Write block 450 len 32768 data: 7 7 6 7 7 8 5 7 23/30 Read block 563 len 32768 24/30 Write block 141 len 36864 data: 5 5 6 7 8 8 9 6 7 25/30 Read block 541 len 36864 26/30 Read block 321 len 40960 27/30 Write block 496 len 40960 data: 8 8 8 9 9 8 7 6 6 5 28/30 Write block 98 len 24576 data: 7 9 6 6 8 10 29/30 Write block 176 len 24576 data: 7 6 6 6 4 4 30/30 Read block 429 len 28672 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 3 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 6 Done Sent Sent false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 20 18 18 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1009 1009 1009 Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:33:26.627285971Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1483} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 110 110 110 F F F 1 020-039 74 74 74 110 110 110 T T T 2 040-059 72 72 72 109 109 109 T T T 3 060-079 72 72 72 109 109 109 T T T 4 080-099 72 72 72 109 109 109 T T T 5 100-119 74 74 74 110 110 110 T T T 6 120-139 72 72 72 109 109 109 F F F 7 140-159 66 66 66 101 101 101 T T T 8 160-179 71 71 71 108 108 108 T T T 9 180-199 71 71 71 108 108 108 T T T 10 200-219 71 71 71 108 108 108 F F F 11 220-239 72 72 72 109 109 109 F F F 12 240-259 74 74 74 110 110 110 F F F 13 260-279 68 68 68 104 104 104 F F F 14 280-299 58 58 58 89 89 89 F F F 15 300-319 74 74 74 110 110 110 F F F 16 320-339 74 74 74 110 110 110 F F F 17 340-359 58 58 58 89 89 89 F F F 18 360-379 66 66 66 100 100 100 F F F 19 380-399 72 72 72 109 109 109 F F F 20 400-419 68 68 68 104 104 104 F F F 21 420-439 63 63 63 97 97 97 F F F 22 440-459 72 72 72 109 109 109 T T T 23 460-479 71 71 71 108 108 108 T T T 24 480-499 72 72 72 109 109 109 T T T 25 500-519 71 71 71 108 108 108 T T T 26 520-539 72 72 72 109 109 109 F F F 27 540-559 64 64 64 98 98 98 T T T 28 560-579 70 70 70 107 107 107 F F F 29 580-599 71 71 71 108 108 108 F F F Max gen: 74, Max flush: 110 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 110 110 110 F F F 1 020-039 75 75 75 111 111 111 F F F 2 040-059 75 75 75 111 111 111 F F F 3 060-079 75 75 75 111 111 111 F F F 4 080-099 75 75 75 111 111 111 F F F 5 100-119 75 75 75 111 111 111 F F F 6 120-139 72 72 72 109 109 109 F F F 7 140-159 75 75 75 111 111 111 F F F 8 160-179 75 75 75 111 111 111 F F F 9 180-199 75 75 75 111 111 111 F F F 10 200-219 71 71 71 108 108 108 F F F 11 220-239 72 72 72 109 109 109 F F F 12 240-259 74 74 74 110 110 110 F F F 13 260-279 68 68 68 104 104 104 F F F 14 280-299 58 58 58 89 89 89 F F F 15 300-319 74 74 74 110 110 110 F F F 16 320-339 74 74 74 110 110 110 F F F 17 340-359 58 58 58 89 89 89 F F F 18 360-379 66 66 66 100 100 100 F F F 19 380-399 72 72 72 109 109 109 F F F 20 400-419 68 68 68 104 104 104 F F F 21 420-439 63 63 63 97 97 97 F F F 22 440-459 75 75 75 111 111 111 F F F 23 460-479 75 75 75 111 111 111 F F F 24 480-499 75 75 75 111 111 111 F F F 25 500-519 75 75 75 111 111 111 F F F 26 520-539 72 72 72 109 109 109 F F F 27 540-559 75 75 75 111 111 111 F F F 28 560-579 70 70 70 107 107 107 F F F 29 580-599 71 71 71 108 108 108 F F F Max gen: 75, Max flush: 111 Begin loop 38 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:30.941575603Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:30.985708609Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:30.98645508Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:30.986500223Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:30.986518701Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:30.986538138Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491} {"msg":"Crucible aa12cfba-3fc9-4405-be89-eeba45cd604f has session id: 2f1f2298-f16b-468f-b086-f0526139f6b4","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:30.986959296Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"Upstairs opts: Upstairs UUID: aa12cfba-3fc9-4405-be89-eeba45cd604f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:30.987033696Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"Crucible stats registered with UUID: aa12cfba-3fc9-4405-be89-eeba45cd604f","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:30.987059742Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.017317034Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.017353078Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491} {"msg":"aa12cfba-3fc9-4405-be89-eeba45cd604f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.017388783Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.017405941Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"client":"0","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.017433257Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"client":"0","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.017452204Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"client":"1","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.017469461Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"client":"1","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.017494208Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"client":"2","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.017528343Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"client":"2","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.017561548Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"":"io task","client":"2","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.017597823Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"":"io task","client":"0","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.017633717Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"":"io task","client":"1","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:63540)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.017668732Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"":"io task","client":"1","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:42781)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.017701458Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"":"io task","client":"2","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:40919)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.017777667Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"":"io task","client":"0","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.018484433Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"client":"0","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.018517379Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"client":"0","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.018539395Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"client":"1","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.018566451Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"client":"1","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.018811076Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"client":"2","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.018839401Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"client":"2","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.019299174Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"client":"1","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.019367534Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"client":"0","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.01960306Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"client":"2","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.019908245Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"client":"0","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.019936371Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"client":"0","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.019952469Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.019965517Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"client":"1","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.019980625Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"client":"1","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.019997722Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.020198653Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"client":"2","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.020222979Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"client":"2","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"[0]R flush_numbers[0..12]: [110, 111, 111, 111, 111, 111, 109, 111, 111, 111, 108, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.020237867Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"":"downstairs","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"[0]R generation[0..12]: [74, 75, 75, 75, 75, 75, 72, 75, 75, 75, 71, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.020252105Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"":"downstairs","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.020266623Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"":"downstairs","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"[1]R flush_numbers[0..12]: [110, 111, 111, 111, 111, 111, 109, 111, 111, 111, 108, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.020280641Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"":"downstairs","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"[1]R generation[0..12]: [74, 75, 75, 75, 75, 75, 72, 75, 75, 75, 71, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.020294439Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"":"downstairs","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.020308607Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"":"downstairs","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"[2]R flush_numbers[0..12]: [110, 111, 111, 111, 111, 111, 109, 111, 111, 111, 108, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.020322515Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"":"downstairs","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"[2]R generation[0..12]: [74, 75, 75, 75, 75, 75, 72, 75, 75, 75, 71, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.020336673Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"":"downstairs","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.020350851Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"":"downstairs","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.020364349Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"":"downstairs","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.020377727Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"":"downstairs","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"Wait for a query_work_queue command to finish before sending IO msg":"Next flush: 112","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.020402843Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"":"downstairs","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.020416841Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"":"downstairs","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.020430369Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.020442627Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"client":"0","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.020456225Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"client":"1","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.020469763Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"client":"2","":"downstairs","session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.020483012Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.02049551Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"aa12cfba-3fc9-4405-be89-eeba45cd604f is now active with session: 2f1f2298-f16b-468f-b086-f0526139f6b4","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.020508718Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.020521186Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491,"session_id":"2f1f2298-f16b-468f-b086-f0526139f6b4"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:31.020533654Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 387 len 4096 data: 6 02/30 Write block 59 len 8192 data: 10 9 03/30 Write block 342 len 40960 data: 8 6 7 8 7 5 4 5 5 7 04/30 Read block 464 len 28672 05/30 Read block 150 len 16384 06/30 Read block 193 len 24576 07/30 Read block 103 len 20480 08/30 Flush 09/30 Write block 253 len 8192 data: 10 10 10/30 Flush 11/30 Write block 99 len 16384 data: 10 7 7 9 12/30 Read block 137 len 32768 13/30 Write block 243 len 36864 data: 10 10 12 9 10 9 7 8 8 14/30 Flush 15/30 Write block 95 len 32768 data: 5 5 6 8 11 8 8 10 16/30 Write block 80 len 4096 data: 9 17/30 Write block 123 len 40960 data: 11 8 8 7 8 6 7 8 9 10 18/30 Write block 443 len 36864 data: 6 6 6 6 6 7 7 8 8 19/30 Write block 164 len 16384 data: 10 10 9 9 20/30 Write block 249 len 8192 data: 8 9 21/30 Read block 88 len 36864 22/30 Read block 495 len 28672 23/30 Flush 24/30 Flush 25/30 Write block 40 len 32768 data: 6 6 8 7 6 7 9 7 26/30 Read block 123 len 28672 27/30 Write block 95 len 40960 data: 6 6 7 9 12 9 9 11 11 9 28/30 Read block 248 len 20480 29/30 Read block 370 len 20480 30/30 Write block 561 len 12288 data: 3 6 6 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 8 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:33:31.03939818Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1491} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 110 110 110 F F F 1 020-039 75 75 75 111 111 111 F F F 2 040-059 76 76 76 112 112 112 T T T 3 060-079 76 76 76 112 112 112 F F F 4 080-099 76 76 76 115 115 115 T T T 5 100-119 76 76 76 115 115 115 T T T 6 120-139 76 76 76 115 115 115 F F F 7 140-159 75 75 75 111 111 111 F F F 8 160-179 76 76 76 115 115 115 F F F 9 180-199 75 75 75 111 111 111 F F F 10 200-219 71 71 71 108 108 108 F F F 11 220-239 72 72 72 109 109 109 F F F 12 240-259 76 76 76 115 115 115 F F F 13 260-279 68 68 68 104 104 104 F F F 14 280-299 58 58 58 89 89 89 F F F 15 300-319 74 74 74 110 110 110 F F F 16 320-339 74 74 74 110 110 110 F F F 17 340-359 76 76 76 112 112 112 F F F 18 360-379 66 66 66 100 100 100 F F F 19 380-399 76 76 76 112 112 112 F F F 20 400-419 68 68 68 104 104 104 F F F 21 420-439 63 63 63 97 97 97 F F F 22 440-459 76 76 76 115 115 115 F F F 23 460-479 75 75 75 111 111 111 F F F 24 480-499 75 75 75 111 111 111 F F F 25 500-519 75 75 75 111 111 111 F F F 26 520-539 72 72 72 109 109 109 F F F 27 540-559 75 75 75 111 111 111 F F F 28 560-579 70 70 70 107 107 107 T T T 29 580-599 71 71 71 108 108 108 F F F Max gen: 76, Max flush: 115 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 110 110 110 F F F 1 020-039 75 75 75 111 111 111 F F F 2 040-059 77 77 77 116 116 116 F F F 3 060-079 76 76 76 112 112 112 F F F 4 080-099 77 77 77 116 116 116 F F F 5 100-119 77 77 77 116 116 116 F F F 6 120-139 76 76 76 115 115 115 F F F 7 140-159 75 75 75 111 111 111 F F F 8 160-179 76 76 76 115 115 115 F F F 9 180-199 75 75 75 111 111 111 F F F 10 200-219 71 71 71 108 108 108 F F F 11 220-239 72 72 72 109 109 109 F F F 12 240-259 76 76 76 115 115 115 F F F 13 260-279 68 68 68 104 104 104 F F F 14 280-299 58 58 58 89 89 89 F F F 15 300-319 74 74 74 110 110 110 F F F 16 320-339 74 74 74 110 110 110 F F F 17 340-359 76 76 76 112 112 112 F F F 18 360-379 66 66 66 100 100 100 F F F 19 380-399 76 76 76 112 112 112 F F F 20 400-419 68 68 68 104 104 104 F F F 21 420-439 63 63 63 97 97 97 F F F 22 440-459 76 76 76 115 115 115 F F F 23 460-479 75 75 75 111 111 111 F F F 24 480-499 75 75 75 111 111 111 F F F 25 500-519 75 75 75 111 111 111 F F F 26 520-539 72 72 72 109 109 109 F F F 27 540-559 75 75 75 111 111 111 F F F 28 560-579 77 77 77 116 116 116 F F F 29 580-599 71 71 71 108 108 108 F F F Max gen: 77, Max flush: 116 Begin loop 39 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.305655816Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:33:35.347631967Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.348010701Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.379356115Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.379955217Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.379999091Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.380018538Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.380037055Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499} {"msg":"Crucible 96f36bc2-1976-410a-9d21-1f68e283f4a6 has session id: 8f927670-fb67-4a3b-a226-d850b5aa7614","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.380462913Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"Upstairs opts: Upstairs UUID: 96f36bc2-1976-410a-9d21-1f68e283f4a6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.380508096Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"Crucible stats registered with UUID: 96f36bc2-1976-410a-9d21-1f68e283f4a6","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.380538002Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.410791945Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.410829789Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499} {"msg":"96f36bc2-1976-410a-9d21-1f68e283f4a6 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.410857905Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.410875452Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"client":"0","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.410903148Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"client":"0","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.410921826Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"client":"1","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.410939423Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"client":"1","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.410956701Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"client":"2","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.410977198Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"client":"2","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.411008913Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"":"io task","client":"2","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.411044718Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"":"io task","client":"0","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.411080302Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"":"io task","client":"1","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"ds_connection connected from Ok(127.0.0.1:52050)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.411125676Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"":"io task","client":"2","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"ds_connection connected from Ok(127.0.0.1:60397)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.411238089Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"":"io task","client":"1","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"ds_connection connected from Ok(127.0.0.1:33779)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.411497131Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"":"io task","client":"0","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.41192238Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"client":"1","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.411954905Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"client":"1","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.411972542Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"client":"2","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.41198769Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"client":"2","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.412133119Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"client":"0","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.412180692Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"client":"0","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.412638185Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"client":"1","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.412682218Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"client":"2","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.412728292Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"client":"0","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.413306907Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"client":"1","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.413347032Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"client":"1","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.413375527Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.413397064Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"client":"2","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.413420961Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"client":"2","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.413445277Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.413468784Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"client":"0","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.413496979Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"client":"0","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"[0]R flush_numbers[0..12]: [110, 111, 116, 112, 116, 116, 115, 111, 115, 111, 108, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.413521046Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"":"downstairs","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"[0]R generation[0..12]: [74, 75, 77, 76, 77, 77, 76, 75, 76, 75, 71, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.413546162Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"":"downstairs","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.413574548Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"":"downstairs","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"[1]R flush_numbers[0..12]: [110, 111, 116, 112, 116, 116, 115, 111, 115, 111, 108, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.413600824Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"":"downstairs","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"[1]R generation[0..12]: [74, 75, 77, 76, 77, 77, 76, 75, 76, 75, 71, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.413627751Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"":"downstairs","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:33:35.413657986Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"":"downstairs","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"[2]R flush_numbers[0..12]: [110, 111, 116, 112, 116, 116, 115, 111, 115, 111, 108, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.413743423Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"":"downstairs","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"[2]R generation[0..12]: [74, 75, 77, 76, 77, 77, 76, 75, 76, 75, 71, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.413780058Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"":"downstairs","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.413810784Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"":"downstairs","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.413831801Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"":"downstairs","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.413854947Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"":"downstairs","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"Next flush: 117","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.413881163Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"":"downstairs","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.41390741Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"":"downstairs","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.413928607Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.413941735Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"client":"0","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.413964231Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"client":"1","":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.413992227Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"client":"2","":"downstairs","session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.414015944Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.41403806Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"96f36bc2-1976-410a-9d21-1f68e283f4a6 is now active with session: 8f927670-fb67-4a3b-a226-d850b5aa7614","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.414060467Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.414082244Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499,"session_id":"8f927670-fb67-4a3b-a226-d850b5aa7614"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:35.414103351Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 370 len 8192 data: 8 7 02/30 Write block 259 len 36864 data: 9 8 6 8 8 7 5 6 5 03/30 Write block 266 len 40960 data: 7 6 6 4 6 8 5 6 5 5 04/30 Read block 344 len 40960 05/30 Read block 436 len 16384 06/30 Read block 285 len 20480 07/30 Read block 423 len 32768 08/30 Write block 63 len 12288 data: 10 12 9 09/30 Read block 293 len 16384 10/30 Write block 51 len 4096 data: 11 11/30 Read block 469 len 28672 12/30 Read block 589 len 28672 13/30 Write block 12 len 36864 data: 6 8 8 9 9 8 9 11 9 14/30 Read block 529 len 36864 15/30 Write block 176 len 20480 data: 8 7 7 7 5 16/30 Flush 17/30 Write block 172 len 20480 data: 4 3 4 5 9 18/30 Write block 130 len 4096 data: 9 19/30 Read block 294 len 16384 20/30 Read block 264 len 4096 21/30 Read block 581 len 4096 22/30 Read block 25 len 20480 23/30 Write block 507 len 4096 data: 5 24/30 Write block 65 len 20480 data: 10 11 10 10 9 25/30 Read block 304 len 4096 26/30 Write block 470 len 16384 data: 7 7 7 7 27/30 Write block 12 len 4096 data: 7 28/30 Read block 287 len 40960 29/30 Read block 129 len 8192 30/30 Write block 528 len 32768 data: 9 10 8 8 8 9 7 6 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 5 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:33:35.432592351Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1499} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 117 117 117 T T T 1 020-039 78 78 78 117 117 117 F F F 2 040-059 78 78 78 117 117 117 F F F 3 060-079 78 78 78 117 117 117 T T T 4 080-099 77 77 77 116 116 116 F F F 5 100-119 77 77 77 116 116 116 F F F 6 120-139 76 76 76 115 115 115 T T T 7 140-159 75 75 75 111 111 111 F F F 8 160-179 78 78 78 117 117 117 T T T 9 180-199 78 78 78 117 117 117 F F F 10 200-219 71 71 71 108 108 108 F F F 11 220-239 72 72 72 109 109 109 F F F 12 240-259 78 78 78 117 117 117 F F F 13 260-279 78 78 78 117 117 117 F F F 14 280-299 58 58 58 89 89 89 F F F 15 300-319 74 74 74 110 110 110 F F F 16 320-339 74 74 74 110 110 110 F F F 17 340-359 76 76 76 112 112 112 F F F 18 360-379 78 78 78 117 117 117 F F F 19 380-399 76 76 76 112 112 112 F F F 20 400-419 68 68 68 104 104 104 F F F 21 420-439 63 63 63 97 97 97 F F F 22 440-459 76 76 76 115 115 115 F F F 23 460-479 75 75 75 111 111 111 T T T 24 480-499 75 75 75 111 111 111 F F F 25 500-519 75 75 75 111 111 111 T T T 26 520-539 72 72 72 109 109 109 T T T 27 540-559 75 75 75 111 111 111 F F F 28 560-579 77 77 77 116 116 116 F F F 29 580-599 71 71 71 108 108 108 F F F Max gen: 78, Max flush: 117 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 79 79 79 118 118 118 F F F 1 020-039 78 78 78 117 117 117 F F F 2 040-059 78 78 78 117 117 117 F F F 3 060-079 79 79 79 118 118 118 F F F 4 080-099 77 77 77 116 116 116 F F F 5 100-119 77 77 77 116 116 116 F F F 6 120-139 79 79 79 118 118 118 F F F 7 140-159 75 75 75 111 111 111 F F F 8 160-179 79 79 79 118 118 118 F F F 9 180-199 78 78 78 117 117 117 F F F 10 200-219 71 71 71 108 108 108 F F F 11 220-239 72 72 72 109 109 109 F F F 12 240-259 78 78 78 117 117 117 F F F 13 260-279 78 78 78 117 117 117 F F F 14 280-299 58 58 58 89 89 89 F F F 15 300-319 74 74 74 110 110 110 F F F 16 320-339 74 74 74 110 110 110 F F F 17 340-359 76 76 76 112 112 112 F F F 18 360-379 78 78 78 117 117 117 F F F 19 380-399 76 76 76 112 112 112 F F F 20 400-419 68 68 68 104 104 104 F F F 21 420-439 63 63 63 97 97 97 F F F 22 440-459 76 76 76 115 115 115 F F F 23 460-479 79 79 79 118 118 118 F F F 24 480-499 75 75 75 111 111 111 F F F 25 500-519 79 79 79 118 118 118 F F F 26 520-539 79 79 79 118 118 118 F F F 27 540-559 75 75 75 111 111 111 F F F 28 560-579 77 77 77 116 116 116 F F F 29 580-599 71 71 71 108 108 108 F F F Max gen: 79, Max flush: 118 Begin loop 40 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.743344901Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.786883283Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.787498403Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.787533718Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.787547956Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.787563493Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507} {"msg":"Crucible 420d344a-e32d-4243-a767-1d9d17359375 has session id: 519d4d92-23c7-48c1-8364-549592e7ea13","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.787968214Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"Upstairs opts: Upstairs UUID: 420d344a-e32d-4243-a767-1d9d17359375, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.788020057Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"Crucible stats registered with UUID: 420d344a-e32d-4243-a767-1d9d17359375","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.788039174Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.818432136Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.818464581Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507} {"msg":"420d344a-e32d-4243-a767-1d9d17359375 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.818517104Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.818537421Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"client":"0","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.818564687Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"client":"0","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.818582714Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"client":"1","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.818599902Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"client":"1","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.818616749Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"client":"2","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.818633577Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"client":"2","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.818650164Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"":"io task","client":"2","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.818667502Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"":"io task","client":"1","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.818690448Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"":"io task","client":"0","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"ds_connection connected from Ok(127.0.0.1:35732)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.818802052Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"":"io task","client":"1","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"ds_connection connected from Ok(127.0.0.1:65143)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.818850995Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"":"io task","client":"2","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"ds_connection connected from Ok(127.0.0.1:62354)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.81888585Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"":"io task","client":"0","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.819623602Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"client":"0","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.819665066Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"client":"0","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.819702051Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"client":"1","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.819731346Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"client":"1","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.819758942Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"client":"2","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.819788228Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"client":"2","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.820262379Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"client":"1","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.820336198Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"client":"0","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.820364724Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"client":"2","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.820835665Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"client":"1","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.820877119Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"client":"1","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.820892257Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.820959097Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"client":"0","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.820980274Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"client":"0","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.820994212Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.82100693Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"client":"2","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.821020788Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"client":"2","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"[0]R flush_numbers[0..12]: [118, 117, 117, 118, 116, 116, 118, 111, 118, 117, 108, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.821035016Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"":"downstairs","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"[0]R generation[0..12]: [79, 78, 78, 79, 77, 77, 79, 75, 79, 78, 71, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.821053293Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"":"downstairs","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.821068021Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"":"downstairs","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"[1]R flush_numbers[0..12]: [118, 117, 117, 118, 116, 116, 118, 111, 118, 117, 108, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.821082629Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"":"downstairs","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"[1]R generation[0..12]: [79, 78, 78, 79, 77, 77, 79, 75, 79, 78, 71, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.821096277Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"":"downstairs","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.821110445Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"":"downstairs","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"[2]R flush_numbers[0..12]: [118, 117, 117, 118, 116, 116, 118, 111, 118, 117, 108, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.821124423Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"":"downstairs","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"[2]R generation[0..12]: [79, 78, 78, 79, 77, 77, 79, 75, 79, 78, 71, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.821138121Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"":"downstairs","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.821152279Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"":"downstairs","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.821165487Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"":"downstairs","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.821178925Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"":"downstairs","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"Next flush: 119","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.821192133Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"":"downstairs","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.821216929Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"":"downstairs","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.821242896Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.821267502Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"client":"0","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.821297978Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"client":"1","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.821328743Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"client":"2","":"downstairs","session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.821358599Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.821386025Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"420d344a-e32d-4243-a767-1d9d17359375 is now active with session: 519d4d92-23c7-48c1-8364-549592e7ea13","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.82141474Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-03T08:33:39.821440487Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507,"session_id":"519d4d92-23c7-48c1-8364-549592e7ea13"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:39.821467563Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 454 len 36864 data: 8 9 6 8 9 8 7 6 5 02/30 Write block 17 len 4096 data: 9 03/30 Read block 540 len 40960 04/30 Read block 388 len 4096 05/30 Read block 321 len 36864 06/30 Write block 446 len 36864 data: 7 7 8 8 9 9 7 8 9 07/30 Read block 228 len 12288 08/30 Write block 243 len 24576 data: 11 11 13 10 11 10 09/30 Read block 460 len 36864 10/30 Read block 550 len 16384 11/30 Write block 256 len 36864 data: 9 9 9 10 9 7 9 9 8 12/30 Write block 583 len 4096 data: 9 13/30 Write block 568 len 24576 data: 7 7 5 5 4 5 14/30 Read block 574 len 24576 15/30 Write block 374 len 16384 data: 5 5 5 5 16/30 Read block 48 len 4096 17/30 Read block 509 len 12288 18/30 Write block 401 len 20480 data: 7 6 6 6 6 19/30 Write block 20 len 20480 data: 10 10 7 7 8 20/30 Flush 21/30 Write block 424 len 28672 data: 4 6 5 6 7 6 5 22/30 Read block 521 len 36864 23/30 Read block 281 len 32768 24/30 Write block 430 len 28672 data: 6 5 6 4 4 7 7 25/30 Read block 57 len 8192 26/30 Read block 361 len 4096 27/30 Read block 190 len 28672 28/30 Write block 417 len 12288 data: 11 9 9 29/30 Read block 285 len 28672 30/30 Write block 494 len 16384 data: 8 8 9 9 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 7 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:33:39.840954587Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1507} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 119 119 119 F F F 1 020-039 80 80 80 119 119 119 F F F 2 040-059 78 78 78 117 117 117 F F F 3 060-079 79 79 79 118 118 118 F F F 4 080-099 77 77 77 116 116 116 F F F 5 100-119 77 77 77 116 116 116 F F F 6 120-139 79 79 79 118 118 118 F F F 7 140-159 75 75 75 111 111 111 F F F 8 160-179 79 79 79 118 118 118 F F F 9 180-199 78 78 78 117 117 117 F F F 10 200-219 71 71 71 108 108 108 F F F 11 220-239 72 72 72 109 109 109 F F F 12 240-259 80 80 80 119 119 119 F F F 13 260-279 80 80 80 119 119 119 F F F 14 280-299 58 58 58 89 89 89 F F F 15 300-319 74 74 74 110 110 110 F F F 16 320-339 74 74 74 110 110 110 F F F 17 340-359 76 76 76 112 112 112 F F F 18 360-379 80 80 80 119 119 119 F F F 19 380-399 76 76 76 112 112 112 F F F 20 400-419 80 80 80 119 119 119 T T T 21 420-439 63 63 63 97 97 97 T T T 22 440-459 80 80 80 119 119 119 F F F 23 460-479 80 80 80 119 119 119 F F F 24 480-499 75 75 75 111 111 111 T T T 25 500-519 79 79 79 118 118 118 F F F 26 520-539 79 79 79 118 118 118 F F F 27 540-559 75 75 75 111 111 111 F F F 28 560-579 80 80 80 119 119 119 F F F 29 580-599 80 80 80 119 119 119 F F F Max gen: 80, Max flush: 119 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 119 119 119 F F F 1 020-039 80 80 80 119 119 119 F F F 2 040-059 78 78 78 117 117 117 F F F 3 060-079 79 79 79 118 118 118 F F F 4 080-099 77 77 77 116 116 116 F F F 5 100-119 77 77 77 116 116 116 F F F 6 120-139 79 79 79 118 118 118 F F F 7 140-159 75 75 75 111 111 111 F F F 8 160-179 79 79 79 118 118 118 F F F 9 180-199 78 78 78 117 117 117 F F F 10 200-219 71 71 71 108 108 108 F F F 11 220-239 72 72 72 109 109 109 F F F 12 240-259 80 80 80 119 119 119 F F F 13 260-279 80 80 80 119 119 119 F F F 14 280-299 58 58 58 89 89 89 F F F 15 300-319 74 74 74 110 110 110 F F F 16 320-339 74 74 74 110 110 110 F F F 17 340-359 76 76 76 112 112 112 F F F 18 360-379 80 80 80 119 119 119 F F F 19 380-399 76 76 76 112 112 112 F F F 20 400-419 81 81 81 120 120 120 F F F 21 420-439 81 81 81 120 120 120 F F F 22 440-459 80 80 80 119 119 119 F F F 23 460-479 80 80 80 119 119 119 F F F 24 480-499 81 81 81 120 120 120 F F F 25 500-519 79 79 79 118 118 118 F F F 26 520-539 79 79 79 118 118 118 F F F 27 540-559 75 75 75 111 111 111 F F F 28 560-579 80 80 80 119 119 119 F F F 29 580-599 80 80 80 119 119 119 F F F Max gen: 81, Max flush: 120 Begin loop 41 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.098085727Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.141466912Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.142063045Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.14209758Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.142111998Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.142126996Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515} {"msg":"Crucible 25c2a12f-25af-4cbe-918f-149ebcc69c32 has session id: 1827c2ae-30f4-4b4e-954d-ba2ac5f77d96","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.142589029Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"Upstairs opts: Upstairs UUID: 25c2a12f-25af-4cbe-918f-149ebcc69c32, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.142626893Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"Crucible stats registered with UUID: 25c2a12f-25af-4cbe-918f-149ebcc69c32","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.142642621Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.173113702Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.173146417Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515} {"msg":"25c2a12f-25af-4cbe-918f-149ebcc69c32 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.17319539Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.173215307Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"client":"0","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.173242023Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"client":"0","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.17326552Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"client":"1","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.173283717Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"client":"1","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.173300634Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"client":"2","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.173317372Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"client":"2","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.173333699Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.173350957Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"":"io task","client":"0","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.173368105Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"":"io task","client":"1","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"ds_connection connected from Ok(127.0.0.1:46616)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.173543009Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"ds_connection connected from Ok(127.0.0.1:54454)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.173582683Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"":"io task","client":"0","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"ds_connection connected from Ok(127.0.0.1:50448)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.173683499Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"":"io task","client":"1","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.174344112Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"client":"2","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.174387296Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"client":"2","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.174420081Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"client":"0","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.174450316Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"client":"0","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.174480582Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"client":"1","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.174512927Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"client":"1","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.174983299Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"client":"2","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.175071576Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"client":"1","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.175260608Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"client":"0","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.175634174Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"client":"1","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.175675928Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"client":"1","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.175708963Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.17572673Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"client":"2","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.175741068Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"client":"2","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.175754666Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.175766974Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"client":"0","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.175780652Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"client":"0","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"[0]R flush_numbers[0..12]: [119, 119, 117, 118, 116, 116, 118, 111, 118, 117, 108, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.17579472Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"":"downstairs","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"[0]R generation[0..12]: [80, 80, 78, 79, 77, 77, 79, 75, 79, 78, 71, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.175808698Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"":"downstairs","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.175822966Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"":"downstairs","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"[1]R flush_numbers[0..12]: [119, 119, 117, 118, 116, 116, 118, 111, 118, 117, 108, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.175840733Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"":"downstairs","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"[1]R generation[0..12]: [80, 80, 78, 79, 77, 77, 79, 75, 79, 78, 71, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.175854901Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"":"downstairs","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.175868949Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"":"downstairs","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"[2]R flush_numbers[0..12]: [119, 119, 117, 118, 116, 116, 118, 111, 118, 117, 108, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.175882717Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"":"downstairs","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"[2]R generation[0..12]: [80, 80, 78, 79, 77, 77, 79, 75, 79, 78, 71, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.175896425Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"":"downstairs","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.175910563Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"":"downstairs","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.175923561Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"":"downstairs","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.17593673Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"":"downstairs","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"Next flush: 121","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.175949617Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"":"downstairs","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.175962616Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"":"downstairs","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2025-01-03T08:33:44.175988182Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.176011399Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"client":"0","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.176033575Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"client":"1","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.176051993Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"client":"2","":"downstairs","session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.176077279Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.176098016Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"25c2a12f-25af-4cbe-918f-149ebcc69c32 is now active with session: 1827c2ae-30f4-4b4e-954d-ba2ac5f77d96","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.176111814Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.176124132Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515,"session_id":"1827c2ae-30f4-4b4e-954d-ba2ac5f77d96"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:44.17613668Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 499 len 12288 data: 10 10 9 02/30 Write block 460 len 20480 data: 8 7 6 4 5 03/30 Write block 520 len 4096 data: 7 04/30 Read block 334 len 8192 05/30 Read block 344 len 16384 06/30 Write block 122 len 32768 data: 10 12 9 9 8 9 7 8 07/30 Write block 225 len 32768 data: 5 6 6 7 6 7 6 7 08/30 Write block 372 len 24576 data: 8 7 6 6 6 6 09/30 Flush 10/30 Flush 11/30 Write block 142 len 32768 data: 6 7 8 9 9 10 7 8 12/30 Read block 7 len 4096 13/30 Write block 97 len 40960 data: 8 10 13 10 10 12 12 10 8 10 14/30 Write block 484 len 16384 data: 5 4 3 3 15/30 Read block 52 len 28672 16/30 Read block 324 len 20480 17/30 Read block 356 len 4096 18/30 Flush 19/30 Write block 2 len 8192 data: 2 2 20/30 Write block 93 len 36864 data: 6 5 7 7 9 11 14 11 11 21/30 Read block 327 len 16384 22/30 Read block 120 len 40960 23/30 Write block 508 len 20480 data: 5 5 4 4 6 24/30 Read block 489 len 36864 25/30 Read block 510 len 32768 26/30 Read block 292 len 24576 27/30 Write block 367 len 8192 data: 6 7 28/30 Read block 232 len 40960 29/30 Write block 572 len 8192 data: 5 6 30/30 Write block 409 len 24576 data: 7 8 10 10 11 14 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 2 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:33:44.195211885Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1515} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 119 119 119 T T T 1 020-039 80 80 80 119 119 119 F F F 2 040-059 78 78 78 117 117 117 F F F 3 060-079 79 79 79 118 118 118 F F F 4 080-099 82 82 82 123 123 123 T T T 5 100-119 82 82 82 123 123 123 T T T 6 120-139 82 82 82 121 121 121 F F F 7 140-159 82 82 82 123 123 123 F F F 8 160-179 79 79 79 118 118 118 F F F 9 180-199 78 78 78 117 117 117 F F F 10 200-219 71 71 71 108 108 108 F F F 11 220-239 82 82 82 121 121 121 F F F 12 240-259 80 80 80 119 119 119 F F F 13 260-279 80 80 80 119 119 119 F F F 14 280-299 58 58 58 89 89 89 F F F 15 300-319 74 74 74 110 110 110 F F F 16 320-339 74 74 74 110 110 110 F F F 17 340-359 76 76 76 112 112 112 F F F 18 360-379 82 82 82 121 121 121 T T T 19 380-399 76 76 76 112 112 112 F F F 20 400-419 81 81 81 120 120 120 T T T 21 420-439 81 81 81 120 120 120 F F F 22 440-459 80 80 80 119 119 119 F F F 23 460-479 82 82 82 121 121 121 F F F 24 480-499 82 82 82 123 123 123 F F F 25 500-519 82 82 82 121 121 121 T T T 26 520-539 82 82 82 121 121 121 F F F 27 540-559 75 75 75 111 111 111 F F F 28 560-579 80 80 80 119 119 119 T T T 29 580-599 80 80 80 119 119 119 F F F Max gen: 82, Max flush: 123 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 124 124 124 F F F 1 020-039 80 80 80 119 119 119 F F F 2 040-059 78 78 78 117 117 117 F F F 3 060-079 79 79 79 118 118 118 F F F 4 080-099 83 83 83 124 124 124 F F F 5 100-119 83 83 83 124 124 124 F F F 6 120-139 82 82 82 121 121 121 F F F 7 140-159 82 82 82 123 123 123 F F F 8 160-179 79 79 79 118 118 118 F F F 9 180-199 78 78 78 117 117 117 F F F 10 200-219 71 71 71 108 108 108 F F F 11 220-239 82 82 82 121 121 121 F F F 12 240-259 80 80 80 119 119 119 F F F 13 260-279 80 80 80 119 119 119 F F F 14 280-299 58 58 58 89 89 89 F F F 15 300-319 74 74 74 110 110 110 F F F 16 320-339 74 74 74 110 110 110 F F F 17 340-359 76 76 76 112 112 112 F F F 18 360-379 83 83 83 124 124 124 F F F 19 380-399 76 76 76 112 112 112 F F F 20 400-419 83 83 83 124 124 124 F F F 21 420-439 81 81 81 120 120 120 F F F 22 440-459 80 80 80 119 119 119 F F F 23 460-479 82 82 82 121 121 121 F F F 24 480-499 82 82 82 123 123 123 F F F 25 500-519 83 83 83 124 124 124 F F F 26 520-539 82 82 82 121 121 121 F F F 27 540-559 75 75 75 111 111 111 F F F 28 560-579 83 83 83 124 124 124 F F F 29 580-599 80 80 80 119 119 119 F F F Max gen: 83, Max flush: 124 Begin loop 42 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.4740146Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:33:48.516281418Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.516656233Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.548312411Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.548984073Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.549027707Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.549043524Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.549058742Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523} {"msg":"Crucible a3631ed8-2b88-4070-9624-5085eee6d646 has session id: 2f713d2f-7e61-4ade-90c8-0ab3f3d369e4","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.549491139Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"Upstairs opts: Upstairs UUID: a3631ed8-2b88-4070-9624-5085eee6d646, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.549535323Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"Crucible stats registered with UUID: a3631ed8-2b88-4070-9624-5085eee6d646","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.54955306Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.580016512Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.580049027Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523} {"msg":"a3631ed8-2b88-4070-9624-5085eee6d646 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.580085072Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.5801024Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"client":"0","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.580130345Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"client":"0","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.580152552Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"client":"1","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.580170739Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"client":"1","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.580198655Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"client":"2","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.580230991Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"client":"2","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.580264516Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.580298701Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.580331276Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"ds_connection connected from Ok(127.0.0.1:41073)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.580424202Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"ds_connection connected from Ok(127.0.0.1:56590)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.580469116Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"ds_connection connected from Ok(127.0.0.1:65395)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.58050765Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.581288316Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"client":"2","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.581316452Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"client":"2","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.581478168Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"client":"1","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.581520532Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"client":"1","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.581582243Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"client":"0","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.581608809Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"client":"0","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.582029238Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"client":"2","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.582054144Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"client":"1","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.582183996Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"client":"0","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.582621612Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"client":"2","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.582647928Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"client":"2","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.582661946Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.582742654Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"client":"1","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.582764811Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"client":"1","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.582778409Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.582791077Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"client":"0","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.582804575Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"client":"0","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"[0]R flush_numbers[0..12]: [124, 119, 117, 118, 124, 124, 121, 123, 118, 117, 108, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.582818513Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"":"downstairs","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"[0]R generation[0..12]: [83, 80, 78, 79, 83, 83, 82, 82, 79, 78, 71, 82]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.582832701Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"":"downstairs","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.582847029Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"":"downstairs","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"[1]R flush_numbers[0..12]: [124, 119, 117, 118, 124, 124, 121, 123, 118, 117, 108, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.582864336Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"":"downstairs","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"[1]R generation[0..12]: [83, 80, 78, 79, 83, 83, 82, 82, 79, 78, 71, 82]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.582878454Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"":"downstairs","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.582893432Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"":"downstairs","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"[2]R flush_numbers[0..12]: [124, 119, 117, 118, 124, 124, 121, 123, 118, 117, 108, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.58290752Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"":"downstairs","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"[2]R generation[0..12]: [83, 80, 78, 79, 83, 83, 82, 82, 79, 78, 71, 82]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.582921298Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"":"downstairs","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.582946324Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"":"downstairs","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:33:48.582973Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"":"downstairs","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.583011775Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"":"downstairs","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"Next flush: 125","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.583026773Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"":"downstairs","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.5830402Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"":"downstairs","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.583053778Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.583066237Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"client":"0","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.583079805Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"client":"1","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.583093043Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"client":"2","":"downstairs","session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.583106351Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.583121209Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"a3631ed8-2b88-4070-9624-5085eee6d646 is now active with session: 2f713d2f-7e61-4ade-90c8-0ab3f3d369e4","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.583134387Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.583146735Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523,"session_id":"2f713d2f-7e61-4ade-90c8-0ab3f3d369e4"} {"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:48.583162642Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 448 len 24576 data: 9 9 10 10 8 9 03/30 Flush 04/30 Write block 24 len 20480 data: 9 6 5 5 5 05/30 Read block 581 len 12288 06/30 Read block 590 len 40960 07/30 Write block 441 len 28672 data: 5 7 7 7 7 8 8 08/30 Read block 491 len 16384 09/30 Write block 346 len 36864 data: 8 6 5 6 6 8 6 5 4 10/30 Read block 15 len 16384 11/30 Read block 60 len 24576 12/30 Write block 506 len 24576 data: 5 6 6 6 5 5 13/30 Write block 371 len 4096 data: 8 14/30 Write block 498 len 16384 data: 9 11 11 10 15/30 Read block 550 len 16384 16/30 Flush 17/30 Write block 309 len 40960 data: 5 6 6 7 7 7 8 7 8 9 18/30 Write block 156 len 36864 data: 7 6 7 8 9 11 12 10 11 19/30 Read block 450 len 40960 20/30 Read block 255 len 12288 21/30 Write block 397 len 28672 data: 8 9 9 9 8 7 7 22/30 Read block 307 len 36864 23/30 Write block 272 len 24576 data: 6 7 6 6 4 5 24/30 Read block 119 len 24576 25/30 Read block 23 len 12288 26/30 Write block 411 len 20480 data: 11 11 12 15 13 27/30 Write block 37 len 4096 data: 4 28/30 Write block 532 len 12288 data: 9 10 8 29/30 Read block 583 len 4096 30/30 Write block 199 len 16384 data: 7 7 6 6 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 10 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:33:48.603048129Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1523} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 124 124 124 F F F 1 020-039 84 84 84 127 127 127 T T T 2 040-059 78 78 78 117 117 117 F F F 3 060-079 79 79 79 118 118 118 F F F 4 080-099 83 83 83 124 124 124 F F F 5 100-119 83 83 83 124 124 124 F F F 6 120-139 82 82 82 121 121 121 F F F 7 140-159 82 82 82 123 123 123 T T T 8 160-179 79 79 79 118 118 118 T T T 9 180-199 78 78 78 117 117 117 T T T 10 200-219 71 71 71 108 108 108 T T T 11 220-239 82 82 82 121 121 121 F F F 12 240-259 80 80 80 119 119 119 F F F 13 260-279 80 80 80 119 119 119 T T T 14 280-299 58 58 58 89 89 89 F F F 15 300-319 74 74 74 110 110 110 T T T 16 320-339 74 74 74 110 110 110 F F F 17 340-359 84 84 84 127 127 127 F F F 18 360-379 84 84 84 127 127 127 F F F 19 380-399 76 76 76 112 112 112 T T T 20 400-419 83 83 83 124 124 124 T T T 21 420-439 81 81 81 120 120 120 F F F 22 440-459 84 84 84 127 127 127 F F F 23 460-479 82 82 82 121 121 121 F F F 24 480-499 84 84 84 127 127 127 F F F 25 500-519 84 84 84 127 127 127 F F F 26 520-539 82 82 82 121 121 121 T T T 27 540-559 75 75 75 111 111 111 F F F 28 560-579 83 83 83 124 124 124 F F F 29 580-599 80 80 80 119 119 119 F F F Max gen: 84, Max flush: 127 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 124 124 124 F F F 1 020-039 85 85 85 128 128 128 F F F 2 040-059 78 78 78 117 117 117 F F F 3 060-079 79 79 79 118 118 118 F F F 4 080-099 83 83 83 124 124 124 F F F 5 100-119 83 83 83 124 124 124 F F F 6 120-139 82 82 82 121 121 121 F F F 7 140-159 85 85 85 128 128 128 F F F 8 160-179 85 85 85 128 128 128 F F F 9 180-199 85 85 85 128 128 128 F F F 10 200-219 85 85 85 128 128 128 F F F 11 220-239 82 82 82 121 121 121 F F F 12 240-259 80 80 80 119 119 119 F F F 13 260-279 85 85 85 128 128 128 F F F 14 280-299 58 58 58 89 89 89 F F F 15 300-319 85 85 85 128 128 128 F F F 16 320-339 74 74 74 110 110 110 F F F 17 340-359 84 84 84 127 127 127 F F F 18 360-379 84 84 84 127 127 127 F F F 19 380-399 85 85 85 128 128 128 F F F 20 400-419 85 85 85 128 128 128 F F F 21 420-439 81 81 81 120 120 120 F F F 22 440-459 84 84 84 127 127 127 F F F 23 460-479 82 82 82 121 121 121 F F F 24 480-499 84 84 84 127 127 127 F F F 25 500-519 84 84 84 127 127 127 F F F 26 520-539 85 85 85 128 128 128 F F F 27 540-559 75 75 75 111 111 111 F F F 28 560-579 83 83 83 124 124 124 F F F 29 580-599 80 80 80 119 119 119 F F F Max gen: 85, Max flush: 128 Begin loop 43 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:52.932356918Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:52.9765226Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:52.977243494Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:52.977279729Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:52.977294287Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:52.977311814Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531} {"msg":"Crucible aa0c0783-0e95-42df-980c-a2300ed05c97 has session id: edbdf54c-e201-4cca-95ff-a22f6b0170cf","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:52.977744631Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"Upstairs opts: Upstairs UUID: aa0c0783-0e95-42df-980c-a2300ed05c97, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:52.977800953Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"Crucible stats registered with UUID: aa0c0783-0e95-42df-980c-a2300ed05c97","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:52.977829419Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.00836696Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.008407404Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531} {"msg":"aa0c0783-0e95-42df-980c-a2300ed05c97 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.008426631Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.008442999Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"client":"0","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.008470355Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"client":"0","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.008488582Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"client":"1","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.008506279Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"client":"1","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.008529216Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"client":"2","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.008547453Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"client":"2","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.00857156Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"":"io task","client":"2","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.008605065Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"":"io task","client":"0","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.00863847Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"":"io task","client":"1","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:41727)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.008670346Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"":"io task","client":"0","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:43477)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.008814794Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"":"io task","client":"2","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:48303)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.008965083Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"":"io task","client":"1","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.009514712Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"client":"1","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.009543278Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"client":"1","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.009686017Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"client":"2","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.009721572Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"client":"2","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.009748398Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"client":"0","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.009775634Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"client":"0","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.010207851Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"client":"1","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.010243676Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"client":"0","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.010575337Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"client":"2","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.010790316Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"client":"1","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.010813833Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"client":"1","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.01082855Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.01090166Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"client":"0","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.010922037Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"client":"0","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.010936085Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.01110198Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"client":"2","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.011121568Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"client":"2","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"[0]R flush_numbers[0..12]: [124, 128, 117, 118, 124, 124, 121, 128, 128, 128, 128, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.011136585Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"":"downstairs","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"[0]R generation[0..12]: [83, 85, 78, 79, 83, 83, 82, 85, 85, 85, 85, 82]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.011150603Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"":"downstairs","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.011164771Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"":"downstairs","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"[1]R flush_numbers[0..12]: [124, 128, 117, 118, 124, 124, 121, 128, 128, 128, 128, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.011178969Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"":"downstairs","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"[1]R generation[0..12]: [83, 85, 78, 79, 83, 83, 82, 85, 85, 85, 85, 82]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.011192607Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"":"downstairs","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.011211404Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"":"downstairs","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"[2]R flush_numbers[0..12]: [124, 128, 117, 118, 124, 124, 121, 128, 128, 128, 128, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.011225882Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"":"downstairs","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"[2]R generation[0..12]: [83, 85, 78, 79, 83, 83, 82, 85, 85, 85, 85, 82]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.01123958Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"":"downstairs","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.011253748Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"":"downstairs","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.011266946Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"":"downstairs","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.011280274Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"":"downstairs","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"Next flush: 129","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.011293452Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"":"downstairs","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.01130641Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.011334417Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.011359183Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"client":"0","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.011383269Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"client":"1","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.011405676Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"client":"2","":"downstairs","session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.011431732Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.011454519Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"aa0c0783-0e95-42df-980c-a2300ed05c97 is now active with session: edbdf54c-e201-4cca-95ff-a22f6b0170cf","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.011479735Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.011509611Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531,"session_id":"edbdf54c-e201-4cca-95ff-a22f6b0170cf"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:53.011539487Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 563 len 40960 data: 7 8 6 7 7 8 8 6 6 6 02/30 Read block 322 len 24576 03/30 Read block 141 len 32768 04/30 Read block 13 len 32768 05/30 Write block 253 len 32768 data: 11 11 8 10 10 10 11 10 06/30 Read block 422 len 4096 07/30 Flush 08/30 Write block 177 len 20480 data: 8 8 8 6 5 09/30 Read block 379 len 32768 10/30 Read block 349 len 32768 11/30 Write block 440 len 28672 data: 5 6 8 8 8 8 9 12/30 Read block 503 len 32768 13/30 Read block 53 len 12288 14/30 Flush 15/30 Write block 169 len 4096 data: 5 16/30 Write block 478 len 24576 data: 6 6 7 6 6 6 17/30 Write block 72 len 32768 data: 7 7 8 9 10 9 9 9 18/30 Read block 187 len 24576 19/30 Read block 49 len 32768 20/30 Read block 440 len 32768 21/30 Write block 72 len 24576 data: 8 8 9 10 11 10 22/30 Write block 123 len 32768 data: 13 10 10 9 10 8 9 10 23/30 Write block 459 len 4096 data: 9 24/30 Read block 454 len 4096 25/30 Read block 18 len 16384 26/30 Write block 466 len 36864 data: 3 4 3 7 8 8 8 8 5 27/30 Read block 5 len 36864 28/30 Write block 233 len 24576 data: 8 10 10 10 11 10 29/30 Write block 261 len 40960 data: 8 10 10 9 6 8 7 7 5 7 30/30 Write block 592 len 24576 data: 8 8 8 7 5 3 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 1 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:33:53.031965174Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1531} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 124 124 124 F F F 1 020-039 85 85 85 128 128 128 F F F 2 040-059 78 78 78 117 117 117 F F F 3 060-079 79 79 79 118 118 118 T T T 4 080-099 83 83 83 124 124 124 F F F 5 100-119 83 83 83 124 124 124 F F F 6 120-139 82 82 82 121 121 121 T T T 7 140-159 85 85 85 128 128 128 F F F 8 160-179 86 86 86 130 130 130 T T T 9 180-199 86 86 86 130 130 130 F F F 10 200-219 85 85 85 128 128 128 F F F 11 220-239 82 82 82 121 121 121 T T T 12 240-259 86 86 86 129 129 129 F F F 13 260-279 86 86 86 129 129 129 T T T 14 280-299 58 58 58 89 89 89 F F F 15 300-319 85 85 85 128 128 128 F F F 16 320-339 74 74 74 110 110 110 F F F 17 340-359 84 84 84 127 127 127 F F F 18 360-379 84 84 84 127 127 127 F F F 19 380-399 85 85 85 128 128 128 F F F 20 400-419 85 85 85 128 128 128 F F F 21 420-439 81 81 81 120 120 120 F F F 22 440-459 86 86 86 130 130 130 T T T 23 460-479 82 82 82 121 121 121 T T T 24 480-499 84 84 84 127 127 127 T T T 25 500-519 84 84 84 127 127 127 F F F 26 520-539 85 85 85 128 128 128 F F F 27 540-559 75 75 75 111 111 111 F F F 28 560-579 86 86 86 129 129 129 F F F 29 580-599 80 80 80 119 119 119 T T T Max gen: 86, Max flush: 130 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 124 124 124 F F F 1 020-039 85 85 85 128 128 128 F F F 2 040-059 78 78 78 117 117 117 F F F 3 060-079 87 87 87 131 131 131 F F F 4 080-099 83 83 83 124 124 124 F F F 5 100-119 83 83 83 124 124 124 F F F 6 120-139 87 87 87 131 131 131 F F F 7 140-159 85 85 85 128 128 128 F F F 8 160-179 87 87 87 131 131 131 F F F 9 180-199 86 86 86 130 130 130 F F F 10 200-219 85 85 85 128 128 128 F F F 11 220-239 87 87 87 131 131 131 F F F 12 240-259 86 86 86 129 129 129 F F F 13 260-279 87 87 87 131 131 131 F F F 14 280-299 58 58 58 89 89 89 F F F 15 300-319 85 85 85 128 128 128 F F F 16 320-339 74 74 74 110 110 110 F F F 17 340-359 84 84 84 127 127 127 F F F 18 360-379 84 84 84 127 127 127 F F F 19 380-399 85 85 85 128 128 128 F F F 20 400-419 85 85 85 128 128 128 F F F 21 420-439 81 81 81 120 120 120 F F F 22 440-459 87 87 87 131 131 131 F F F 23 460-479 87 87 87 131 131 131 F F F 24 480-499 87 87 87 131 131 131 F F F 25 500-519 84 84 84 127 127 127 F F F 26 520-539 85 85 85 128 128 128 F F F 27 540-559 75 75 75 111 111 111 F F F 28 560-579 86 86 86 129 129 129 F F F 29 580-599 87 87 87 131 131 131 F F F Max gen: 87, Max flush: 131 Begin loop 44 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.320905659Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.364240161Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.364887267Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.364931171Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.364953357Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.364972685Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539} {"msg":"Crucible a8deacb9-c799-4a79-81d6-0cc222054c95 has session id: 187094bd-2d11-4da4-a076-3ce7e0d29f88","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.365561859Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"Upstairs opts: Upstairs UUID: a8deacb9-c799-4a79-81d6-0cc222054c95, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.365605662Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"Crucible stats registered with UUID: a8deacb9-c799-4a79-81d6-0cc222054c95","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.365626319Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.396043598Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.396075513Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539} {"msg":"a8deacb9-c799-4a79-81d6-0cc222054c95 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.396125496Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.396145823Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"client":"0","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.396172529Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"client":"0","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.396190436Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"client":"1","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.396208064Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"client":"1","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.396224971Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"client":"2","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.396246078Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"client":"2","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.396264365Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"":"io task","client":"2","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.396297141Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"":"io task","client":"0","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.396332506Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"":"io task","client":"1","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"ds_connection connected from Ok(127.0.0.1:33360)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.39636945Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"":"io task","client":"2","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"ds_connection connected from Ok(127.0.0.1:52624)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.396405325Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"":"io task","client":"0","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"ds_connection connected from Ok(127.0.0.1:47563)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.396600586Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"":"io task","client":"1","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.39712651Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"client":"2","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.397168833Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"client":"2","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.397201589Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"client":"0","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.397230584Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"client":"0","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.39726035Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"client":"1","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.397288496Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"client":"1","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.397852864Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"client":"2","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.397895057Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"client":"0","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.397926293Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"client":"1","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.398458125Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"client":"1","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.39849719Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"client":"1","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.398514657Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.398527845Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"client":"2","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.398541903Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"client":"2","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.398555621Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.398568019Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"client":"0","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.398581397Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"client":"0","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"[0]R flush_numbers[0..12]: [124, 128, 117, 131, 124, 124, 131, 128, 131, 130, 128, 131]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.398595325Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"":"downstairs","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"[0]R generation[0..12]: [83, 85, 78, 87, 83, 83, 87, 85, 87, 86, 85, 87]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.398609273Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"":"downstairs","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.398623471Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"":"downstairs","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"[1]R flush_numbers[0..12]: [124, 128, 117, 131, 124, 124, 131, 128, 131, 130, 128, 131]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.398637519Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"":"downstairs","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"[1]R generation[0..12]: [83, 85, 78, 87, 83, 83, 87, 85, 87, 86, 85, 87]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.398651277Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"":"downstairs","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.398665435Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"":"downstairs","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"[2]R flush_numbers[0..12]: [124, 128, 117, 131, 124, 124, 131, 128, 131, 130, 128, 131]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.398682552Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"":"downstairs","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"[2]R generation[0..12]: [83, 85, 78, 87, 83, 83, 87, 85, 87, 86, 85, 87]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.39869713Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"":"downstairs","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.398711438Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"":"downstairs","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.398724416Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"":"downstairs","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.398737794Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"":"downstairs","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"Next flush: 132","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.398759671Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"":"downstairs","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.398772719Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"":"downstairs","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.398786207Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.398798335Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"client":"0","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.398811644Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"client":"1","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.398824702Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"client":"2","":"downstairs","session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.398838Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.398850578Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"a8deacb9-c799-4a79-81d6-0cc222054c95 is now active with session: 187094bd-2d11-4da4-a076-3ce7e0d29f88","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.398863516Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.398875554Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539,"session_id":"187094bd-2d11-4da4-a076-3ce7e0d29f88"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:33:57.398887922Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 360 len 8192 data: 6 7 02/30 Write block 457 len 8192 data: 9 10 03/30 Write block 111 len 32768 data: 3 4 5 6 6 7 8 10 04/30 Read block 48 len 36864 05/30 Read block 101 len 16384 06/30 Write block 12 len 4096 data: 8 07/30 Write block 365 len 40960 data: 8 7 7 8 7 9 9 9 8 7 08/30 Read block 67 len 4096 09/30 Read block 476 len 24576 10/30 Read block 590 len 20480 11/30 Write block 24 len 8192 data: 10 7 12/30 Write block 450 len 4096 data: 11 13/30 Write block 108 len 16384 data: 9 8 4 4 14/30 Flush 15/30 Write block 67 len 8192 data: 11 11 16/30 Write block 25 len 24576 data: 8 6 6 6 6 7 17/30 Read block 492 len 24576 18/30 Read block 530 len 24576 19/30 Write block 296 len 8192 data: 7 7 20/30 Read block 539 len 4096 21/30 Write block 276 len 32768 data: 5 6 4 2 3 3 4 5 22/30 Write block 261 len 36864 data: 9 11 11 10 7 9 8 8 6 23/30 Read block 324 len 12288 24/30 Write block 386 len 28672 data: 4 7 5 7 8 11 12 25/30 Read block 247 len 16384 26/30 Flush 27/30 Write block 14 len 36864 data: 9 10 10 10 10 12 11 11 8 28/30 Write block 401 len 24576 data: 9 8 8 7 7 7 29/30 Read block 270 len 8192 30/30 Write block 462 len 28672 data: 7 5 6 5 4 5 4 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 9 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:33:57.416542105Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1539} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 132 132 132 T T T 1 020-039 88 88 88 133 133 133 T T T 2 040-059 78 78 78 117 117 117 F F F 3 060-079 88 88 88 133 133 133 F F F 4 080-099 83 83 83 124 124 124 F F F 5 100-119 88 88 88 132 132 132 F F F 6 120-139 87 87 87 131 131 131 F F F 7 140-159 85 85 85 128 128 128 F F F 8 160-179 87 87 87 131 131 131 F F F 9 180-199 86 86 86 130 130 130 F F F 10 200-219 85 85 85 128 128 128 F F F 11 220-239 87 87 87 131 131 131 F F F 12 240-259 86 86 86 129 129 129 F F F 13 260-279 88 88 88 133 133 133 F F F 14 280-299 88 88 88 133 133 133 F F F 15 300-319 85 85 85 128 128 128 F F F 16 320-339 74 74 74 110 110 110 F F F 17 340-359 84 84 84 127 127 127 F F F 18 360-379 88 88 88 132 132 132 F F F 19 380-399 88 88 88 133 133 133 F F F 20 400-419 85 85 85 128 128 128 T T T 21 420-439 81 81 81 120 120 120 F F F 22 440-459 88 88 88 132 132 132 F F F 23 460-479 87 87 87 131 131 131 T T T 24 480-499 87 87 87 131 131 131 F F F 25 500-519 84 84 84 127 127 127 F F F 26 520-539 85 85 85 128 128 128 F F F 27 540-559 75 75 75 111 111 111 F F F 28 560-579 86 86 86 129 129 129 F F F 29 580-599 87 87 87 131 131 131 F F F Max gen: 88, Max flush: 133 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 134 134 134 F F F 1 020-039 89 89 89 134 134 134 F F F 2 040-059 78 78 78 117 117 117 F F F 3 060-079 88 88 88 133 133 133 F F F 4 080-099 83 83 83 124 124 124 F F F 5 100-119 88 88 88 132 132 132 F F F 6 120-139 87 87 87 131 131 131 F F F 7 140-159 85 85 85 128 128 128 F F F 8 160-179 87 87 87 131 131 131 F F F 9 180-199 86 86 86 130 130 130 F F F 10 200-219 85 85 85 128 128 128 F F F 11 220-239 87 87 87 131 131 131 F F F 12 240-259 86 86 86 129 129 129 F F F 13 260-279 88 88 88 133 133 133 F F F 14 280-299 88 88 88 133 133 133 F F F 15 300-319 85 85 85 128 128 128 F F F 16 320-339 74 74 74 110 110 110 F F F 17 340-359 84 84 84 127 127 127 F F F 18 360-379 88 88 88 132 132 132 F F F 19 380-399 88 88 88 133 133 133 F F F 20 400-419 89 89 89 134 134 134 F F F 21 420-439 81 81 81 120 120 120 F F F 22 440-459 88 88 88 132 132 132 F F F 23 460-479 89 89 89 134 134 134 F F F 24 480-499 87 87 87 131 131 131 F F F 25 500-519 84 84 84 127 127 127 F F F 26 520-539 85 85 85 128 128 128 F F F 27 540-559 75 75 75 111 111 111 F F F 28 560-579 86 86 86 129 129 129 F F F 29 580-599 87 87 87 131 131 131 F F F Max gen: 89, Max flush: 134 Begin loop 45 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.682014955Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:34:01.724091911Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.724471356Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.755663831Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.756235798Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.756282821Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.756305558Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.756324725Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547} {"msg":"Crucible 8ea41bbf-4e84-4490-8ba0-edc69177e85a has session id: e94c7f3d-a1af-4208-abdb-e4fa6b4bc794","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.756697031Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"Upstairs opts: Upstairs UUID: 8ea41bbf-4e84-4490-8ba0-edc69177e85a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.756752683Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"Crucible stats registered with UUID: 8ea41bbf-4e84-4490-8ba0-edc69177e85a","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.756782958Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.787264936Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.7873101Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547} {"msg":"8ea41bbf-4e84-4490-8ba0-edc69177e85a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.787341905Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.787372491Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"client":"0","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.787418294Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"client":"0","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.787457108Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"client":"1","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.787493403Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"client":"1","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.787528818Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"client":"2","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.787563463Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"client":"2","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.787590679Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"":"io task","client":"2","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.787629703Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"":"io task","client":"0","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.787665718Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"":"io task","client":"1","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"ds_connection connected from Ok(127.0.0.1:52410)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.787702043Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"":"io task","client":"2","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"ds_connection connected from Ok(127.0.0.1:60948)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.787828674Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"":"io task","client":"0","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"ds_connection connected from Ok(127.0.0.1:42218)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.787868028Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"":"io task","client":"1","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.788582354Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"client":"1","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.788649054Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"client":"1","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.78867784Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"client":"2","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.788700237Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"client":"2","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.788727153Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"client":"0","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.788754509Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"client":"0","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.789232319Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"client":"1","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.789275583Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"client":"2","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.789311068Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"client":"0","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.789847939Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"client":"1","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.789887853Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"client":"1","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.789915459Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.789933547Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"client":"2","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.789948064Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"client":"2","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.789962162Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.789974831Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"client":"0","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.789988729Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"client":"0","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"[0]R flush_numbers[0..12]: [134, 134, 117, 133, 124, 132, 131, 128, 131, 130, 128, 131]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.790002827Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"":"downstairs","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"[0]R generation[0..12]: [89, 89, 78, 88, 83, 88, 87, 85, 87, 86, 85, 87]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.790016805Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"":"downstairs","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.790031092Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"":"downstairs","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"[1]R flush_numbers[0..12]: [134, 134, 117, 133, 124, 132, 131, 128, 131, 130, 128, 131]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.790044901Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"":"downstairs","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"[1]R generation[0..12]: [89, 89, 78, 88, 83, 88, 87, 85, 87, 86, 85, 87]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.790058708Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"":"downstairs","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.790072866Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"":"downstairs","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"[2]R flush_numbers[0..12]: [134, 134, 117, 133, 124, 132, 131, 128, 131, 130, 128, 131]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.790094753Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"":"downstairs","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R generation[0..12]: [89, 89, 78, 88, 83, 88, 87, 85, 87, 86, 85, 87]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.790172882Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"":"downstairs","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.790212806Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"":"downstairs","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.790242222Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"":"downstairs","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.790271108Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"":"downstairs","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"Next flush: 135","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.790299413Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"":"downstairs","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.790328049Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"":"downstairs","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.790357195Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.79038722Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"client":"0","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.790410037Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"client":"1","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.790424225Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"client":"2","":"downstairs","session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.790437743Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.790450341Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"8ea41bbf-4e84-4490-8ba0-edc69177e85a is now active with session: e94c7f3d-a1af-4208-abdb-e4fa6b4bc794","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.790463369Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.790475798Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547,"session_id":"e94c7f3d-a1af-4208-abdb-e4fa6b4bc794"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:01.790488116Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 261 len 24576 data: 10 12 12 11 8 10 02/30 Read block 40 len 8192 03/30 Write block 323 len 32768 data: 6 6 6 6 6 5 5 5 04/30 Write block 310 len 20480 data: 7 7 8 8 8 05/30 Read block 459 len 32768 06/30 Read block 450 len 12288 07/30 Read block 131 len 40960 08/30 Write block 175 len 4096 data: 6 09/30 Write block 330 len 12288 data: 6 4 4 10/30 Write block 438 len 4096 data: 5 11/30 Read block 495 len 8192 12/30 Read block 82 len 20480 13/30 Write block 63 len 12288 data: 11 13 11 14/30 Read block 342 len 36864 15/30 Read block 468 len 36864 16/30 Read block 39 len 12288 17/30 Read block 578 len 20480 18/30 Read block 158 len 24576 19/30 Read block 194 len 16384 20/30 Write block 171 len 12288 data: 5 5 4 21/30 Read block 63 len 20480 22/30 Read block 395 len 8192 23/30 Write block 545 len 16384 data: 7 8 9 10 24/30 Read block 484 len 20480 25/30 Read block 290 len 24576 26/30 Write block 228 len 24576 data: 8 7 8 7 8 9 27/30 Read block 92 len 8192 28/30 Read block 257 len 12288 29/30 Read block 77 len 20480 30/30 Read block 56 len 12288 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Write 8 Done Done Done false 1003 Acked Write 5 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Read 3 Done Done Done false 1006 Acked Read 10 Done Done Done false 1007 Acked Write 1 Done Done Done false 1008 Acked Write 3 Done Done Done false 1009 Acked Write 1 Done Done Done false 1010 Acked Read 2 Done Done Done false 1011 Acked Read 5 Done Done Done false 1012 Acked Write 3 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 30 30 29 89 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:34:01.809371699Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1547} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 134 134 134 F F F 1 020-039 89 89 89 134 134 134 F F F 2 040-059 78 78 78 117 117 117 F F F 3 060-079 88 88 88 133 133 133 T T T 4 080-099 83 83 83 124 124 124 F F F 5 100-119 88 88 88 132 132 132 F F F 6 120-139 87 87 87 131 131 131 F F F 7 140-159 85 85 85 128 128 128 F F F 8 160-179 87 87 87 131 131 131 T T T 9 180-199 86 86 86 130 130 130 F F F 10 200-219 85 85 85 128 128 128 F F F 11 220-239 87 87 87 131 131 131 T T T 12 240-259 86 86 86 129 129 129 F F F 13 260-279 88 88 88 133 133 133 T T T 14 280-299 88 88 88 133 133 133 F F F 15 300-319 85 85 85 128 128 128 T T T 16 320-339 74 74 74 110 110 110 T T T 17 340-359 84 84 84 127 127 127 F F F 18 360-379 88 88 88 132 132 132 F F F 19 380-399 88 88 88 133 133 133 F F F 20 400-419 89 89 89 134 134 134 F F F 21 420-439 81 81 81 120 120 120 T T T 22 440-459 88 88 88 132 132 132 F F F 23 460-479 89 89 89 134 134 134 F F F 24 480-499 87 87 87 131 131 131 F F F 25 500-519 84 84 84 127 127 127 F F F 26 520-539 85 85 85 128 128 128 F F F 27 540-559 75 75 75 111 111 111 T T T 28 560-579 86 86 86 129 129 129 F F F 29 580-599 87 87 87 131 131 131 F F F Max gen: 89, Max flush: 134 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 134 134 134 F F F 1 020-039 89 89 89 134 134 134 F F F 2 040-059 78 78 78 117 117 117 F F F 3 060-079 90 90 90 135 135 135 F F F 4 080-099 83 83 83 124 124 124 F F F 5 100-119 88 88 88 132 132 132 F F F 6 120-139 87 87 87 131 131 131 F F F 7 140-159 85 85 85 128 128 128 F F F 8 160-179 90 90 90 135 135 135 F F F 9 180-199 86 86 86 130 130 130 F F F 10 200-219 85 85 85 128 128 128 F F F 11 220-239 90 90 90 135 135 135 F F F 12 240-259 86 86 86 129 129 129 F F F 13 260-279 90 90 90 135 135 135 F F F 14 280-299 88 88 88 133 133 133 F F F 15 300-319 90 90 90 135 135 135 F F F 16 320-339 90 90 90 135 135 135 F F F 17 340-359 84 84 84 127 127 127 F F F 18 360-379 88 88 88 132 132 132 F F F 19 380-399 88 88 88 133 133 133 F F F 20 400-419 89 89 89 134 134 134 F F F 21 420-439 90 90 90 135 135 135 F F F 22 440-459 88 88 88 132 132 132 F F F 23 460-479 89 89 89 134 134 134 F F F 24 480-499 87 87 87 131 131 131 F F F 25 500-519 84 84 84 127 127 127 F F F 26 520-539 85 85 85 128 128 128 F F F 27 540-559 90 90 90 135 135 135 F F F 28 560-579 86 86 86 129 129 129 F F F 29 580-599 87 87 87 131 131 131 F F F Max gen: 90, Max flush: 135 Begin loop 46 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.121518814Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:34:06.163282136Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.163669589Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.194592124Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.195147943Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.195196856Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.195215363Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.19523457Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555} {"msg":"Crucible 198901a5-bc71-4feb-88ff-032c7559d627 has session id: 2bbfca24-9a88-4e57-bc0b-b825d235ac41","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.195671537Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"Upstairs opts: Upstairs UUID: 198901a5-bc71-4feb-88ff-032c7559d627, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.195725369Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"Crucible stats registered with UUID: 198901a5-bc71-4feb-88ff-032c7559d627","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.195746936Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.226084036Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.226129539Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555} {"msg":"198901a5-bc71-4feb-88ff-032c7559d627 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.226161865Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.22619198Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"client":"0","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.226239443Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"client":"0","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.226277858Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"client":"1","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.226312693Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"client":"1","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.226347718Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"client":"2","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.226385292Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"client":"2","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.226417367Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.226449593Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.226482388Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"ds_connection connected from Ok(127.0.0.1:60029)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.226515613Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"ds_connection connected from Ok(127.0.0.1:49465)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.226543319Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"ds_connection connected from Ok(127.0.0.1:41223)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.226571755Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.227176706Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"client":"2","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.227209012Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"client":"2","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.227235418Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"client":"0","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.227264144Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"client":"0","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.22729127Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"client":"1","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.227320036Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"client":"1","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.227761111Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"client":"1","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.227894032Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"client":"2","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.22803956Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"client":"0","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.228293873Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"client":"1","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.228322489Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"client":"1","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.228338227Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.228420185Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"client":"2","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.228439902Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"client":"2","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.22845423Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.228636613Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"client":"0","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.228655481Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"client":"0","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"[0]R flush_numbers[0..12]: [134, 134, 117, 135, 124, 132, 131, 128, 135, 130, 128, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.228669659Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"":"downstairs","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"[0]R generation[0..12]: [89, 89, 78, 90, 83, 88, 87, 85, 90, 86, 85, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.228683716Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"":"downstairs","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.228698024Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"":"downstairs","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"[1]R flush_numbers[0..12]: [134, 134, 117, 135, 124, 132, 131, 128, 135, 130, 128, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.228712052Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"":"downstairs","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"[1]R generation[0..12]: [89, 89, 78, 90, 83, 88, 87, 85, 90, 86, 85, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.2287257Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"":"downstairs","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.228739848Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"":"downstairs","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"[2]R flush_numbers[0..12]: [134, 134, 117, 135, 124, 132, 131, 128, 135, 130, 128, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.228757136Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"":"downstairs","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"[2]R generation[0..12]: [89, 89, 78, 90, 83, 88, 87, 85, 90, 86, 85, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.228771004Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"":"downstairs","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.228785012Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"":"downstairs","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"Max found gen is 91","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.22879809Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"":"downstairs","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"Generation requested: 92 >= found:91","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.228811538Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"":"downstairs","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"Next flush: 136","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.228824616Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"":"downstairs","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:34:06.228837724Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"":"downstairs","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.228857801Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.228870099Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"client":"0","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.228885237Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"client":"1","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.228899075Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"client":"2","":"downstairs","session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.228912673Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.228925111Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"198901a5-bc71-4feb-88ff-032c7559d627 is now active with session: 2bbfca24-9a88-4e57-bc0b-b825d235ac41","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.228938099Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.228950318Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555,"session_id":"2bbfca24-9a88-4e57-bc0b-b825d235ac41"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:06.228964715Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 168 len 4096 data: 6 02/30 Read block 286 len 40960 03/30 Write block 517 len 8192 data: 5 6 04/30 Read block 32 len 16384 05/30 Read block 493 len 24576 06/30 Read block 374 len 4096 07/30 Write block 514 len 16384 data: 8 5 6 6 08/30 Write block 38 len 28672 data: 4 7 7 7 9 8 7 09/30 Read block 314 len 24576 10/30 Write block 53 len 16384 data: 8 8 5 7 11/30 Read block 487 len 4096 12/30 Read block 499 len 4096 13/30 Read block 371 len 32768 14/30 Write block 319 len 32768 data: 7 6 6 6 7 7 7 7 15/30 Read block 215 len 28672 16/30 Write block 407 len 40960 data: 6 7 8 9 12 12 13 16 14 12 17/30 Read block 484 len 36864 18/30 Write block 548 len 4096 data: 11 19/30 Read block 37 len 12288 20/30 Write block 353 len 20480 data: 6 5 3 3 5 21/30 Flush 22/30 Write block 240 len 8192 data: 11 10 23/30 Write block 268 len 36864 data: 9 7 8 9 7 8 7 7 6 24/30 Write block 481 len 28672 data: 7 7 7 6 5 4 4 25/30 Write block 80 len 12288 data: 10 9 9 26/30 Write block 429 len 8192 data: 7 7 27/30 Write block 484 len 24576 data: 7 6 5 5 3 4 28/30 Write block 454 len 36864 data: 10 10 7 10 11 10 9 8 8 29/30 Read block 259 len 40960 30/30 Read block 294 len 16384 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 2 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 9 8 8 25 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:34:06.247472483Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1555} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 134 134 134 F F F 1 020-039 92 92 92 136 136 136 F F F 2 040-059 92 92 92 136 136 136 F F F 3 060-079 90 90 90 135 135 135 F F F 4 080-099 83 83 83 124 124 124 T T T 5 100-119 88 88 88 132 132 132 F F F 6 120-139 87 87 87 131 131 131 F F F 7 140-159 85 85 85 128 128 128 F F F 8 160-179 92 92 92 136 136 136 F F F 9 180-199 86 86 86 130 130 130 F F F 10 200-219 85 85 85 128 128 128 F F F 11 220-239 90 90 90 135 135 135 F F F 12 240-259 86 86 86 129 129 129 T T T 13 260-279 90 90 90 135 135 135 T T T 14 280-299 88 88 88 133 133 133 F F F 15 300-319 92 92 92 136 136 136 F F F 16 320-339 92 92 92 136 136 136 F F F 17 340-359 92 92 92 136 136 136 F F F 18 360-379 88 88 88 132 132 132 F F F 19 380-399 88 88 88 133 133 133 F F F 20 400-419 92 92 92 136 136 136 F F F 21 420-439 90 90 90 135 135 135 T T T 22 440-459 88 88 88 132 132 132 T T T 23 460-479 89 89 89 134 134 134 T T T 24 480-499 87 87 87 131 131 131 T T T 25 500-519 92 92 92 136 136 136 F F F 26 520-539 85 85 85 128 128 128 F F F 27 540-559 92 92 92 136 136 136 F F F 28 560-579 86 86 86 129 129 129 F F F 29 580-599 87 87 87 131 131 131 F F F Max gen: 92, Max flush: 136 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 134 134 134 F F F 1 020-039 92 92 92 136 136 136 F F F 2 040-059 92 92 92 136 136 136 F F F 3 060-079 90 90 90 135 135 135 F F F 4 080-099 93 93 93 137 137 137 F F F 5 100-119 88 88 88 132 132 132 F F F 6 120-139 87 87 87 131 131 131 F F F 7 140-159 85 85 85 128 128 128 F F F 8 160-179 92 92 92 136 136 136 F F F 9 180-199 86 86 86 130 130 130 F F F 10 200-219 85 85 85 128 128 128 F F F 11 220-239 90 90 90 135 135 135 F F F 12 240-259 93 93 93 137 137 137 F F F 13 260-279 93 93 93 137 137 137 F F F 14 280-299 88 88 88 133 133 133 F F F 15 300-319 92 92 92 136 136 136 F F F 16 320-339 92 92 92 136 136 136 F F F 17 340-359 92 92 92 136 136 136 F F F 18 360-379 88 88 88 132 132 132 F F F 19 380-399 88 88 88 133 133 133 F F F 20 400-419 92 92 92 136 136 136 F F F 21 420-439 93 93 93 137 137 137 F F F 22 440-459 93 93 93 137 137 137 F F F 23 460-479 93 93 93 137 137 137 F F F 24 480-499 93 93 93 137 137 137 F F F 25 500-519 92 92 92 136 136 136 F F F 26 520-539 85 85 85 128 128 128 F F F 27 540-559 92 92 92 136 136 136 F F F 28 560-579 86 86 86 129 129 129 F F F 29 580-599 87 87 87 131 131 131 F F F Max gen: 93, Max flush: 137 Begin loop 47 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.553322969Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.596439573Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.59707515Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.597111415Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.597131502Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.59714651Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563} {"msg":"Crucible 597b22d3-18b0-4689-85cd-9cf95031ad89 has session id: a63121fb-557a-4dc8-b62e-229aa1643566","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.597619051Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"Upstairs opts: Upstairs UUID: 597b22d3-18b0-4689-85cd-9cf95031ad89, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.597663114Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"Crucible stats registered with UUID: 597b22d3-18b0-4689-85cd-9cf95031ad89","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.59768995Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.62803483Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.628066995Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563} {"msg":"597b22d3-18b0-4689-85cd-9cf95031ad89 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.628094431Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.628111608Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"client":"0","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.628138095Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"client":"0","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.628155972Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"client":"1","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.62817286Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"client":"1","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.628195726Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"client":"2","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.628228342Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"client":"2","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.628261257Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"":"io task","client":"2","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.628288593Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"":"io task","client":"0","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.62831089Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"":"io task","client":"1","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"ds_connection connected from Ok(127.0.0.1:53531)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.628342325Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"":"io task","client":"0","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"ds_connection connected from Ok(127.0.0.1:52196)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.62837787Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"":"io task","client":"1","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"ds_connection connected from Ok(127.0.0.1:38766)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.628410375Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"":"io task","client":"2","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.629141388Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"client":"0","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.629179373Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"client":"0","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.629211738Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"client":"1","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.629240784Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"client":"1","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.629271439Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"client":"2","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.629291026Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"client":"2","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.629798172Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"client":"0","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.629832457Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"client":"1","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.629974656Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"client":"2","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.630342433Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"client":"1","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.630382977Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"client":"1","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.630417962Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.630483082Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"client":"0","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.630511288Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"client":"0","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.630526126Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.630538674Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"client":"2","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.630553082Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"client":"2","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"[0]R flush_numbers[0..12]: [134, 136, 136, 135, 137, 132, 131, 128, 136, 130, 128, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.6305676Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"":"downstairs","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"[0]R generation[0..12]: [89, 92, 92, 90, 93, 88, 87, 85, 92, 86, 85, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.630582078Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"":"downstairs","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.630596566Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"":"downstairs","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"[1]R flush_numbers[0..12]: [134, 136, 136, 135, 137, 132, 131, 128, 136, 130, 128, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.630610594Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"":"downstairs","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"[1]R generation[0..12]: [89, 92, 92, 90, 93, 88, 87, 85, 92, 86, 85, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.630624352Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"":"downstairs","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.63063834Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"":"downstairs","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"[2]R flush_numbers[0..12]: [134, 136, 136, 135, 137, 132, 131, 128, 136, 130, 128, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.630652397Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"":"downstairs","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"[2]R generation[0..12]: [89, 92, 92, 90, 93, 88, 87, 85, 92, 86, 85, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.630666206Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"":"downstairs","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:34:10.630684093Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"":"downstairs","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.63070528Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"":"downstairs","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.630718748Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"":"downstairs","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"Next flush: 138","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.630731846Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"":"downstairs","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.630744954Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"":"downstairs","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.630758242Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.63077029Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"client":"0","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.630783538Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"client":"1","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.630797027Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"client":"2","":"downstairs","session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.630810374Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.630822793Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"597b22d3-18b0-4689-85cd-9cf95031ad89 is now active with session: a63121fb-557a-4dc8-b62e-229aa1643566","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.630835601Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.630847769Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563,"session_id":"a63121fb-557a-4dc8-b62e-229aa1643566"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:10.630861447Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Write block 74 len 24576 data: 10 11 12 11 10 10 04/30 Read block 357 len 28672 05/30 Write block 352 len 28672 data: 7 7 6 4 4 6 7 06/30 Write block 97 len 8192 data: 10 12 07/30 Write block 455 len 16384 data: 11 8 11 12 08/30 Read block 231 len 40960 09/30 Write block 179 len 40960 data: 9 7 6 6 8 9 9 10 10 10 10/30 Read block 238 len 8192 11/30 Write block 182 len 20480 data: 7 9 10 10 11 12/30 Write block 117 len 32768 data: 9 11 12 12 11 11 14 11 13/30 Write block 181 len 32768 data: 7 8 10 11 11 12 11 11 14/30 Write block 394 len 28672 data: 10 8 8 9 10 10 10 15/30 Write block 348 len 12288 data: 6 7 7 16/30 Read block 433 len 20480 17/30 Write block 311 len 40960 data: 8 9 9 9 9 8 9 10 8 7 18/30 Flush 19/30 Write block 555 len 40960 data: 10 7 6 7 5 4 4 7 8 9 20/30 Read block 289 len 4096 21/30 Write block 28 len 16384 data: 7 7 8 6 22/30 Flush 23/30 Write block 548 len 28672 data: 12 9 11 12 13 11 9 24/30 Read block 181 len 8192 25/30 Write block 527 len 28672 data: 12 10 11 9 9 10 11 26/30 Read block 578 len 20480 27/30 Read block 57 len 20480 28/30 Write block 9 len 8192 data: 7 7 29/30 Read block 365 len 36864 30/30 Write block 196 len 32768 data: 6 6 6 8 8 7 7 5 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 7 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:34:10.650546843Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1563} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 134 134 134 T T T 1 020-039 94 94 94 141 141 141 F F F 2 040-059 92 92 92 136 136 136 F F F 3 060-079 94 94 94 140 140 140 F F F 4 080-099 94 94 94 140 140 140 F F F 5 100-119 94 94 94 140 140 140 F F F 6 120-139 94 94 94 140 140 140 F F F 7 140-159 85 85 85 128 128 128 F F F 8 160-179 94 94 94 140 140 140 F F F 9 180-199 94 94 94 140 140 140 T T T 10 200-219 85 85 85 128 128 128 T T T 11 220-239 90 90 90 135 135 135 F F F 12 240-259 93 93 93 137 137 137 F F F 13 260-279 93 93 93 137 137 137 F F F 14 280-299 88 88 88 133 133 133 F F F 15 300-319 94 94 94 140 140 140 F F F 16 320-339 94 94 94 140 140 140 F F F 17 340-359 94 94 94 140 140 140 F F F 18 360-379 88 88 88 132 132 132 F F F 19 380-399 94 94 94 140 140 140 F F F 20 400-419 94 94 94 140 140 140 F F F 21 420-439 93 93 93 137 137 137 F F F 22 440-459 94 94 94 140 140 140 F F F 23 460-479 93 93 93 137 137 137 F F F 24 480-499 93 93 93 137 137 137 F F F 25 500-519 92 92 92 136 136 136 F F F 26 520-539 85 85 85 128 128 128 T T T 27 540-559 94 94 94 141 141 141 T T T 28 560-579 94 94 94 141 141 141 F F F 29 580-599 87 87 87 131 131 131 F F F Max gen: 94, Max flush: 141 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 142 142 142 F F F 1 020-039 94 94 94 141 141 141 F F F 2 040-059 92 92 92 136 136 136 F F F 3 060-079 94 94 94 140 140 140 F F F 4 080-099 94 94 94 140 140 140 F F F 5 100-119 94 94 94 140 140 140 F F F 6 120-139 94 94 94 140 140 140 F F F 7 140-159 85 85 85 128 128 128 F F F 8 160-179 94 94 94 140 140 140 F F F 9 180-199 95 95 95 142 142 142 F F F 10 200-219 95 95 95 142 142 142 F F F 11 220-239 90 90 90 135 135 135 F F F 12 240-259 93 93 93 137 137 137 F F F 13 260-279 93 93 93 137 137 137 F F F 14 280-299 88 88 88 133 133 133 F F F 15 300-319 94 94 94 140 140 140 F F F 16 320-339 94 94 94 140 140 140 F F F 17 340-359 94 94 94 140 140 140 F F F 18 360-379 88 88 88 132 132 132 F F F 19 380-399 94 94 94 140 140 140 F F F 20 400-419 94 94 94 140 140 140 F F F 21 420-439 93 93 93 137 137 137 F F F 22 440-459 94 94 94 140 140 140 F F F 23 460-479 93 93 93 137 137 137 F F F 24 480-499 93 93 93 137 137 137 F F F 25 500-519 92 92 92 136 136 136 F F F 26 520-539 95 95 95 142 142 142 F F F 27 540-559 95 95 95 142 142 142 F F F 28 560-579 94 94 94 141 141 141 F F F 29 580-599 87 87 87 131 131 131 F F F Max gen: 95, Max flush: 142 Begin loop 48 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.919069778Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.962911757Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.963520048Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.963555763Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.963570391Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.963585918Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571} {"msg":"Crucible 104504e6-6d26-479d-8bbd-0b0aa70e431e has session id: bd6ea69d-cb8d-4fdf-b901-291f9a313bbe","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.96398284Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"Upstairs opts: Upstairs UUID: 104504e6-6d26-479d-8bbd-0b0aa70e431e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.964034583Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"Crucible stats registered with UUID: 104504e6-6d26-479d-8bbd-0b0aa70e431e","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.96405507Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.994403339Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.994437694Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571} {"msg":"104504e6-6d26-479d-8bbd-0b0aa70e431e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.994470769Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.994487666Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"client":"0","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.994515512Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"client":"0","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.99453374Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"client":"1","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.994551577Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"client":"1","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.994577133Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"client":"2","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.994612898Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"client":"2","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.994646053Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.994679468Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.994713673Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"ds_connection connected from Ok(127.0.0.1:62946)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.994879589Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"ds_connection connected from Ok(127.0.0.1:35619)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.994948089Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"ds_connection connected from Ok(127.0.0.1:35299)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.994982464Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.995589215Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"client":"2","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.99562717Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"client":"2","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.995655016Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"client":"0","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.995686341Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"client":"0","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.995805104Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"client":"1","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.995837609Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"client":"1","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.996334857Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"client":"2","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.996366732Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"client":"0","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.996452569Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"client":"1","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.996966194Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"client":"0","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.99699409Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"client":"0","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.997010288Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.997023476Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"client":"2","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.997040814Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"client":"2","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.997055751Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.99706821Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"client":"1","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.997081697Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"client":"1","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"[0]R flush_numbers[0..12]: [142, 141, 136, 140, 140, 140, 140, 128, 140, 142, 142, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.997095566Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"":"downstairs","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"[0]R generation[0..12]: [95, 94, 92, 94, 94, 94, 94, 85, 94, 95, 95, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.997109704Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"":"downstairs","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.997124081Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"":"downstairs","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"[1]R flush_numbers[0..12]: [142, 141, 136, 140, 140, 140, 140, 128, 140, 142, 142, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.997137969Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"":"downstairs","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"[1]R generation[0..12]: [95, 94, 92, 94, 94, 94, 94, 85, 94, 95, 95, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.997151797Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"":"downstairs","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.997165705Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"":"downstairs","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"[2]R flush_numbers[0..12]: [142, 141, 136, 140, 140, 140, 140, 128, 140, 142, 142, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.997179573Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"":"downstairs","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"[2]R generation[0..12]: [95, 94, 92, 94, 94, 94, 94, 85, 94, 95, 95, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.997193471Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"":"downstairs","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.997207459Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"":"downstairs","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.997220557Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"":"downstairs","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.997233815Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"":"downstairs","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msgWait for a query_work_queue command to finish before sending IO ":"Next flush: 143","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.997260251Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"":"downstairs","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.997273649Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"":"downstairs","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.997286767Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.997298916Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"client":"0","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.997312304Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"client":"1","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.997325582Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"client":"2","":"downstairs","session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.99733878Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.997351228Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"104504e6-6d26-479d-8bbd-0b0aa70e431e is now active with session: bd6ea69d-cb8d-4fdf-b901-291f9a313bbe","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.997364066Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.997376125Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571,"session_id":"bd6ea69d-cb8d-4fdf-b901-291f9a313bbe"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:14.997388333Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 117 len 12288 data: 10 12 13 02/30 Read block 340 len 12288 03/30 Read block 214 len 12288 04/30 Read block 512 len 20480 05/30 Write block 460 len 40960 data: 10 9 9 6 7 6 5 6 5 8 06/30 Read block 541 len 24576 07/30 Read block 113 len 36864 08/30 Write block 102 len 4096 data: 13 09/30 Write block 137 len 28672 data: 6 6 8 6 6 7 8 10/30 Read block 340 len 28672 11/30 Write block 33 len 8192 data: 5 4 12/30 Flush 13/30 Write block 289 len 4096 data: 7 14/30 Read block 40 len 36864 15/30 Read block 295 len 16384 16/30 Write block 141 len 40960 data: 7 8 9 9 10 10 11 8 9 6 17/30 Write block 440 len 24576 data: 6 7 9 9 9 9 18/30 Read block 233 len 8192 19/30 Flush 20/30 Flush 21/30 Write block 8 len 24576 data: 5 8 8 7 9 9 22/30 Write block 532 len 32768 data: 11 12 9 7 6 6 5 3 23/30 Write block 42 len 20480 data: 10 9 8 8 10 24/30 Read block 12 len 4096 25/30 Read block 578 len 20480 26/30 Read block 545 len 24576 27/30 Write block 406 len 24576 data: 8 7 8 9 10 13 28/30 Read block 421 len 12288 29/30 Read block 445 len 4096 30/30 Write block 90 len 8192 data: 10 7 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 6 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:34:15.016282554Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1571} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 142 142 142 T T T 1 020-039 96 96 96 143 143 143 F F F 2 040-059 92 92 92 136 136 136 T T T 3 060-079 94 94 94 140 140 140 F F F 4 080-099 94 94 94 140 140 140 T T T 5 100-119 96 96 96 143 143 143 F F F 6 120-139 96 96 96 143 143 143 F F F 7 140-159 96 96 96 144 144 144 F F F 8 160-179 94 94 94 140 140 140 F F F 9 180-199 95 95 95 142 142 142 F F F 10 200-219 95 95 95 142 142 142 F F F 11 220-239 90 90 90 135 135 135 F F F 12 240-259 93 93 93 137 137 137 F F F 13 260-279 93 93 93 137 137 137 F F F 14 280-299 96 96 96 144 144 144 F F F 15 300-319 94 94 94 140 140 140 F F F 16 320-339 94 94 94 140 140 140 F F F 17 340-359 94 94 94 140 140 140 F F F 18 360-379 88 88 88 132 132 132 F F F 19 380-399 94 94 94 140 140 140 F F F 20 400-419 94 94 94 140 140 140 T T T 21 420-439 93 93 93 137 137 137 F F F 22 440-459 96 96 96 144 144 144 F F F 23 460-479 96 96 96 143 143 143 F F F 24 480-499 93 93 93 137 137 137 F F F 25 500-519 92 92 92 136 136 136 F F F 26 520-539 95 95 95 142 142 142 T T T 27 540-559 95 95 95 142 142 142 F F F 28 560-579 94 94 94 141 141 141 F F F 29 580-599 87 87 87 131 131 131 F F F Max gen: 96, Max flush: 144 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 145 145 145 F F F 1 020-039 96 96 96 143 143 143 F F F 2 040-059 97 97 97 145 145 145 F F F 3 060-079 94 94 94 140 140 140 F F F 4 080-099 97 97 97 145 145 145 F F F 5 100-119 96 96 96 143 143 143 F F F 6 120-139 96 96 96 143 143 143 F F F 7 140-159 96 96 96 144 144 144 F F F 8 160-179 94 94 94 140 140 140 F F F 9 180-199 95 95 95 142 142 142 F F F 10 200-219 95 95 95 142 142 142 F F F 11 220-239 90 90 90 135 135 135 F F F 12 240-259 93 93 93 137 137 137 F F F 13 260-279 93 93 93 137 137 137 F F F 14 280-299 96 96 96 144 144 144 F F F 15 300-319 94 94 94 140 140 140 F F F 16 320-339 94 94 94 140 140 140 F F F 17 340-359 94 94 94 140 140 140 F F F 18 360-379 88 88 88 132 132 132 F F F 19 380-399 94 94 94 140 140 140 F F F 20 400-419 97 97 97 145 145 145 F F F 21 420-439 93 93 93 137 137 137 F F F 22 440-459 96 96 96 144 144 144 F F F 23 460-479 96 96 96 143 143 143 F F F 24 480-499 93 93 93 137 137 137 F F F 25 500-519 92 92 92 136 136 136 F F F 26 520-539 97 97 97 145 145 145 F F F 27 540-559 95 95 95 142 142 142 F F F 28 560-579 94 94 94 141 141 141 F F F 29 580-599 87 87 87 131 131 131 F F F Max gen: 97, Max flush: 145 Begin loop 49 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.287777495Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.331565092Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.332151276Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.332188631Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.332204128Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.332218466Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579} {"msg":"Crucible af1b07b2-9a18-4b5b-9282-ae8dc1f1d39d has session id: 7a8bfe03-7a72-4375-8fe9-21ec627cab58","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.332576244Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"Upstairs opts: Upstairs UUID: af1b07b2-9a18-4b5b-9282-ae8dc1f1d39d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.332620258Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"Crucible stats registered with UUID: af1b07b2-9a18-4b5b-9282-ae8dc1f1d39d","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.332645604Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.363403953Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.363455905Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579} {"msg":"af1b07b2-9a18-4b5b-9282-ae8dc1f1d39d active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.363487761Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.363518216Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"client":"0","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.363565419Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"client":"0","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.363602114Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"client":"1","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.363635699Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"client":"1","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.363667245Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"client":"2","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.36370006Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"client":"2","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.363723826Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"":"io task","client":"2","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.363748853Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"":"io task","client":"0","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.363772129Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"":"io task","client":"1","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"ds_connection connected from Ok(127.0.0.1:62954)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.363797375Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"":"io task","client":"2","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"ds_connection connected from Ok(127.0.0.1:53755)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.363942314Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"":"io task","client":"1","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"ds_connection connected from Ok(127.0.0.1:48815)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.363997166Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"":"io task","client":"0","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.36465373Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"client":"0","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.364707453Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"client":"0","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.364738338Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"client":"1","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.364765884Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"client":"1","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.364913823Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"client":"2","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.364936639Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"client":"2","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.365399481Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"client":"0","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.365431577Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"client":"1","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.365448054Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"client":"2","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.366005683Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"client":"1","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.366047517Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"client":"1","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.366077462Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.366100159Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"client":"2","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.366123686Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"client":"2","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.366147072Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.366170339Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"client":"0","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.366194905Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"client":"0","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"[0]R flush_numbers[0..12]: [145, 143, 145, 140, 145, 143, 143, 144, 140, 142, 142, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.366222401Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"":"downstairs","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"[0]R generation[0..12]: [97, 96, 97, 94, 97, 96, 96, 96, 94, 95, 95, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.366248428Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"":"downstairs","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.366276673Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"":"downstairs","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"[1]R flush_numbers[0..12]: [145, 143, 145, 140, 145, 143, 143, 144, 140, 142, 142, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.366303589Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"":"downstairs","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"[1]R generation[0..12]: [97, 96, 97, 94, 97, 96, 96, 96, 94, 95, 95, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.366330276Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"":"downstairs","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {Wait for a query_work_queue command to finish before sending IO "msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.36636904Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"":"downstairs","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"[2]R flush_numbers[0..12]: [145, 143, 145, 140, 145, 143, 143, 144, 140, 142, 142, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.366400865Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"":"downstairs","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"[2]R generation[0..12]: [97, 96, 97, 94, 97, 96, 96, 96, 94, 95, 95, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.366430661Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"":"downstairs","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.366461186Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"":"downstairs","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.366490072Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"":"downstairs","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.366519678Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"":"downstairs","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"Next flush: 146","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.366548504Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"":"downstairs","":"downstairs"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.366581279Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"":"downstairs","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.366604696Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.366617744Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"client":"0","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.366631812Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"client":"1","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.3666455Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"client":"2","":"downstairs","session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.366659148Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.366671896Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"af1b07b2-9a18-4b5b-9282-ae8dc1f1d39d is now active with session: 7a8bfe03-7a72-4375-8fe9-21ec627cab58","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.366685014Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.366697222Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579,"session_id":"7a8bfe03-7a72-4375-8fe9-21ec627cab58"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:19.36670959Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 243 len 40960 data: 12 12 14 11 12 11 9 10 9 11 02/30 Flush 03/30 Write block 500 len 28672 data: 12 11 8 7 7 6 6 04/30 Write block 176 len 16384 data: 10 9 9 10 05/30 Read block 91 len 20480 06/30 Read block 578 len 16384 07/30 Write block 444 len 16384 data: 10 10 10 9 08/30 Read block 327 len 36864 09/30 Flush 10/30 Write block 370 len 40960 data: 10 10 10 9 8 7 7 7 6 6 11/30 Write block 480 len 12288 data: 8 8 8 12/30 Flush 13/30 Write block 17 len 28672 data: 11 11 13 12 12 9 8 14/30 Read block 107 len 32768 15/30 Read block 232 len 16384 16/30 Read block 449 len 40960 17/30 Write block 421 len 8192 data: 7 4 18/30 Write block 54 len 40960 data: 9 6 8 10 11 11 10 8 10 12 19/30 Read block 254 len 8192 20/30 Write block 333 len 24576 data: 5 6 5 5 5 6 21/30 Write block 473 len 16384 data: 9 6 7 9 22/30 Read block 158 len 4096 23/30 Write block 501 len 32768 data: 12 9 8 8 7 7 7 7 24/30 Read block 291 len 8192 25/30 Read block 108 len 4096 26/30 Read block 313 len 12288 27/30 Read block 387 len 12288 28/30 Read block 154 len 20480 29/30 Write block 15 len 16384 data: 11 11 12 12 30/30 Write block 572 len 8192 data: 7 7 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 7 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:34:19.385465352Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1579} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 145 145 145 T T T 1 020-039 96 96 96 143 143 143 T T T 2 040-059 97 97 97 145 145 145 T T T 3 060-079 94 94 94 140 140 140 T T T 4 080-099 97 97 97 145 145 145 F F F 5 100-119 96 96 96 143 143 143 F F F 6 120-139 96 96 96 143 143 143 F F F 7 140-159 96 96 96 144 144 144 F F F 8 160-179 98 98 98 147 147 147 F F F 9 180-199 95 95 95 142 142 142 F F F 10 200-219 95 95 95 142 142 142 F F F 11 220-239 90 90 90 135 135 135 F F F 12 240-259 98 98 98 146 146 146 F F F 13 260-279 93 93 93 137 137 137 F F F 14 280-299 96 96 96 144 144 144 F F F 15 300-319 94 94 94 140 140 140 F F F 16 320-339 94 94 94 140 140 140 T T T 17 340-359 94 94 94 140 140 140 F F F 18 360-379 98 98 98 148 148 148 F F F 19 380-399 94 94 94 140 140 140 F F F 20 400-419 97 97 97 145 145 145 F F F 21 420-439 93 93 93 137 137 137 T T T 22 440-459 98 98 98 147 147 147 F F F 23 460-479 96 96 96 143 143 143 T T T 24 480-499 98 98 98 148 148 148 F F F 25 500-519 98 98 98 147 147 147 T T T 26 520-539 97 97 97 145 145 145 F F F 27 540-559 95 95 95 142 142 142 F F F 28 560-579 94 94 94 141 141 141 T T T 29 580-599 87 87 87 131 131 131 F F F Max gen: 98, Max flush: 148 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 149 149 149 F F F 1 020-039 99 99 99 149 149 149 F F F 2 040-059 99 99 99 149 149 149 F F F 3 060-079 99 99 99 149 149 149 F F F 4 080-099 97 97 97 145 145 145 F F F 5 100-119 96 96 96 143 143 143 F F F 6 120-139 96 96 96 143 143 143 F F F 7 140-159 96 96 96 144 144 144 F F F 8 160-179 98 98 98 147 147 147 F F F 9 180-199 95 95 95 142 142 142 F F F 10 200-219 95 95 95 142 142 142 F F F 11 220-239 90 90 90 135 135 135 F F F 12 240-259 98 98 98 146 146 146 F F F 13 260-279 93 93 93 137 137 137 F F F 14 280-299 96 96 96 144 144 144 F F F 15 300-319 94 94 94 140 140 140 F F F 16 320-339 99 99 99 149 149 149 F F F 17 340-359 94 94 94 140 140 140 F F F 18 360-379 98 98 98 148 148 148 F F F 19 380-399 94 94 94 140 140 140 F F F 20 400-419 97 97 97 145 145 145 F F F 21 420-439 99 99 99 149 149 149 F F F 22 440-459 98 98 98 147 147 147 F F F 23 460-479 99 99 99 149 149 149 F F F 24 480-499 98 98 98 148 148 148 F F F 25 500-519 99 99 99 149 149 149 F F F 26 520-539 97 97 97 145 145 145 F F F 27 540-559 95 95 95 142 142 142 F F F 28 560-579 99 99 99 149 149 149 F F F 29 580-599 87 87 87 131 131 131 F F F Max gen: 99, Max flush: 149 Begin loop 50 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.66937691Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.712506112Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.713125052Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.713161246Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.713175694Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.713190032Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587} {"msg":"Crucible ff6f4e3d-05d0-4658-8495-cc64d808e478 has session id: 3853c040-07f6-4f53-ba13-473028fced39","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.713600932Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"Upstairs opts: Upstairs UUID: ff6f4e3d-05d0-4658-8495-cc64d808e478, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.713645176Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"Crucible stats registered with UUID: ff6f4e3d-05d0-4658-8495-cc64d808e478","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.713670622Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.744120966Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.744156961Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587} {"msg":"ff6f4e3d-05d0-4658-8495-cc64d808e478 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.744183207Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.744197695Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"client":"0","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.744221021Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"client":"0","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.744235349Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"client":"1","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.744249057Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"client":"1","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.744262595Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"client":"2","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.744279853Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"client":"2","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.744305969Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"":"io task","client":"2","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.744333265Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"":"io task","client":"0","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.744361501Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"":"io task","client":"1","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"ds_connection connected from Ok(127.0.0.1:63648)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.744426961Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"":"io task","client":"0","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"ds_connection connected from Ok(127.0.0.1:59778)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.744463836Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"":"io task","client":"2","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"ds_connection connected from Ok(127.0.0.1:60414)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.744491742Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"":"io task","client":"1","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.74532469Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"client":"2","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.745354706Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"client":"2","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.745375683Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"client":"1","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.745390431Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"client":"1","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.745404269Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"client":"0","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.745424286Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"client":"0","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.745985184Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"client":"2","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.746138861Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"client":"1","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.746168377Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"client":"0","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.746550191Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"client":"2","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.746579247Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"client":"2","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.746594245Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.746678212Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"client":"0","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.746698449Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"client":"0","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.746712187Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.746749662Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"client":"1","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.74676614Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"client":"1","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"[0]R flush_numbers[0..12]: [149, 149, 149, 149, 145, 143, 143, 144, 147, 142, 142, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.746783167Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"":"downstairs","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"[0]R generation[0..12]: [99, 99, 99, 99, 97, 96, 96, 96, 98, 95, 95, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.746800715Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"":"downstairs","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.746815052Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"":"downstairs","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"[1]R flush_numbers[0..12]: [149, 149, 149, 149, 145, 143, 143, 144, 147, 142, 142, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.746828961Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"":"downstairs","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"[1]R generation[0..12]: [99, 99, 99, 99, 97, 96, 96, 96, 98, 95, 95, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.746842788Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"":"downstairs","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.746856646Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"":"downstairs","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"[2]R flush_numbers[0..12]: [149, 149, 149, 149, 145, 143, 143, 144, 147, 142, 142, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.746870524Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"":"downstairs","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"[2]R generation[0..12]: [99, 99, 99, 99, 97, 96, 96, 96, 98, 95, 95, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.746884222Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"":"downstairs","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.74689814Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"":"downstairs","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.746911298Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"":"downstairs","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.746924536Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"":"downstairs","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"Next flush: 150","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:34:23.746937495Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"":"downstairs","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.746957522Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"":"downstairs","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.74697071Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.746983018Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"client":"0","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.746998376Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"client":"1","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.747012014Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"client":"2","":"downstairs","session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.747025312Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.74703786Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"ff6f4e3d-05d0-4658-8495-cc64d808e478 is now active with session: 3853c040-07f6-4f53-ba13-473028fced39","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.747050838Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.747063076Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587,"session_id":"3853c040-07f6-4f53-ba13-473028fced39"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:23.747075505Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 237 len 12288 data: 12 11 11 02/30 Read block 491 len 36864 03/30 Read block 487 len 4096 04/30 Read block 442 len 8192 05/30 Write block 346 len 20480 data: 9 7 7 8 8 06/30 Write block 492 len 20480 data: 7 9 9 9 10 07/30 Flush 08/30 Write block 316 len 24576 data: 9 10 11 9 8 7 09/30 Read block 17 len 24576 10/30 Read block 398 len 12288 11/30 Write block 406 len 12288 data: 9 8 9 12/30 Write block 98 len 12288 data: 13 15 12 13/30 Read block 387 len 12288 14/30 Read block 130 len 12288 15/30 Read block 146 len 28672 16/30 Read block 487 len 36864 17/30 Read block 71 len 32768 18/30 Read block 46 len 4096 19/30 Write block 267 len 28672 data: 9 10 8 9 10 8 9 20/30 Read block 125 len 32768 21/30 Read block 241 len 24576 22/30 Write block 495 len 20480 data: 10 11 10 10 12 23/30 Read block 445 len 4096 24/30 Read block 108 len 40960 25/30 Write block 210 len 32768 data: 5 6 6 8 9 7 6 7 26/30 Write block 578 len 40960 data: 8 10 10 8 9 10 11 8 9 11 27/30 Write block 49 len 8192 data: 9 11 28/30 Write block 485 len 4096 data: 7 29/30 Read block 317 len 36864 30/30 Write block 237 len 24576 data: 13 12 12 12 11 10 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 6 Done Done Done false 1008 Acked Read 6 Done Done Done false 1009 Acked Read 3 Done Done Done false 1010 Acked Write 3 Done Done Done false 1011 Acked Write 3 Done Done Done false 1012 Acked Read 3 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1006 1006 1006 Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:34:23.765743199Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1587} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 149 149 149 F F F 1 020-039 99 99 99 149 149 149 F F F 2 040-059 99 99 99 149 149 149 T T T 3 060-079 99 99 99 149 149 149 F F F 4 080-099 97 97 97 145 145 145 T T T 5 100-119 96 96 96 143 143 143 T T T 6 120-139 96 96 96 143 143 143 F F F 7 140-159 96 96 96 144 144 144 F F F 8 160-179 98 98 98 147 147 147 F F F 9 180-199 95 95 95 142 142 142 F F F 10 200-219 95 95 95 142 142 142 T T T 11 220-239 100 100 100 150 150 150 T T T 12 240-259 98 98 98 146 146 146 T T T 13 260-279 93 93 93 137 137 137 T T T 14 280-299 96 96 96 144 144 144 F F F 15 300-319 94 94 94 140 140 140 T T T 16 320-339 99 99 99 149 149 149 T T T 17 340-359 100 100 100 150 150 150 F F F 18 360-379 98 98 98 148 148 148 F F F 19 380-399 94 94 94 140 140 140 F F F 20 400-419 97 97 97 145 145 145 T T T 21 420-439 99 99 99 149 149 149 F F F 22 440-459 98 98 98 147 147 147 F F F 23 460-479 99 99 99 149 149 149 F F F 24 480-499 100 100 100 150 150 150 T T T 25 500-519 99 99 99 149 149 149 F F F 26 520-539 97 97 97 145 145 145 F F F 27 540-559 95 95 95 142 142 142 F F F 28 560-579 99 99 99 149 149 149 T T T 29 580-599 87 87 87 131 131 131 T T T Max gen: 100, Max flush: 150 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 149 149 149 F F F 1 020-039 99 99 99 149 149 149 F F F 2 040-059 101 101 101 151 151 151 F F F 3 060-079 99 99 99 149 149 149 F F F 4 080-099 101 101 101 151 151 151 F F F 5 100-119 101 101 101 151 151 151 F F F 6 120-139 96 96 96 143 143 143 F F F 7 140-159 96 96 96 144 144 144 F F F 8 160-179 98 98 98 147 147 147 F F F 9 180-199 95 95 95 142 142 142 F F F 10 200-219 101 101 101 151 151 151 F F F 11 220-239 101 101 101 151 151 151 F F F 12 240-259 101 101 101 151 151 151 F F F 13 260-279 101 101 101 151 151 151 F F F 14 280-299 96 96 96 144 144 144 F F F 15 300-319 101 101 101 151 151 151 F F F 16 320-339 101 101 101 151 151 151 F F F 17 340-359 100 100 100 150 150 150 F F F 18 360-379 98 98 98 148 148 148 F F F 19 380-399 94 94 94 140 140 140 F F F 20 400-419 101 101 101 151 151 151 F F F 21 420-439 99 99 99 149 149 149 F F F 22 440-459 98 98 98 147 147 147 F F F 23 460-479 99 99 99 149 149 149 F F F 24 480-499 101 101 101 151 151 151 F F F 25 500-519 99 99 99 149 149 149 F F F 26 520-539 97 97 97 145 145 145 F F F 27 540-559 95 95 95 142 142 142 F F F 28 560-579 101 101 101 151 151 151 F F F 29 580-599 101 101 101 151 151 151 F F F Max gen: 101, Max flush: 151 Begin loop 51 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.089045065Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.132522217Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.133193119Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.133228704Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.133243162Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.13325775Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595} {"msg":"Crucible 588cba60-a6f0-459c-86ed-0025c5c93512 has session id: bd2dcc97-5f42-46fd-971a-aae237ea7657","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.133644353Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"Upstairs opts: Upstairs UUID: 588cba60-a6f0-459c-86ed-0025c5c93512, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.133687007Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"Crucible stats registered with UUID: 588cba60-a6f0-459c-86ed-0025c5c93512","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.133709754Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.164912638Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.164945293Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595} {"msg":"588cba60-a6f0-459c-86ed-0025c5c93512 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.164975958Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.164993446Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"client":"0","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.165020322Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"client":"0","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.165039489Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"client":"1","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.165068265Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"client":"1","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.16510228Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"client":"2","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.165139085Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"client":"2","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.16517362Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.165208875Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.165246779Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"ds_connection connected from Ok(127.0.0.1:54048)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.165284833Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"ds_connection connected from Ok(127.0.0.1:46777)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.165339685Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"ds_connection connected from Ok(127.0.0.1:34994)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.165409765Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.165941308Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"client":"2","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.165983551Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"client":"2","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.166015057Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"client":"0","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.166050642Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"client":"0","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.166243873Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"client":"1","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.16626939Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"client":"1","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.1668148Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"client":"0","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.166857014Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"client":"1","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.166890069Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"client":"2","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.167424211Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"client":"0","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.167462046Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"client":"0","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.167487322Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.167502689Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"client":"1","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.167516927Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"client":"1","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.167530376Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.167542724Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"client":"2","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.167556402Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"client":"2","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"[0]R flush_numbers[0..12]: [149, 149, 151, 149, 151, 151, 143, 144, 147, 142, 151, 151]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.16757053Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"":"downstairs","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"[0]R generation[0..12]: [99, 99, 101, 99, 101, 101, 96, 96, 98, 95, 101, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.167588047Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"":"downstairs","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.167602685Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"":"downstairs","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"[1]R flush_numbers[0..12]: [149, 149, 151, 149, 151, 151, 143, 144, 147, 142, 151, 151]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.167616733Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"":"downstairs","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"[1]R generation[0..12]: [99, 99, 101, 99, 101, 101, 96, 96, 98, 95, 101, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.167630601Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"":"downstairs","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.167651718Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"":"downstairs","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R flush_numbers[0..12]: [149, 149, 151, 149, 151, 151, 143, 144, 147, 142, 151, 151]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.167685413Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"":"downstairs","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"[2]R generation[0..12]: [99, 99, 101, 99, 101, 101, 96, 96, 98, 95, 101, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.167710309Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"":"downstairs","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.167725787Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"":"downstairs","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.167738845Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"":"downstairs","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.167752163Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"":"downstairs","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"Next flush: 152","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.167765691Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"":"downstairs","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.167778779Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"":"downstairs","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.167792047Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.167804205Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"client":"0","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.167820053Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"client":"1","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.167833461Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"client":"2","":"downstairs","session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.167846849Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.167859168Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"588cba60-a6f0-459c-86ed-0025c5c93512 is now active with session: bd2dcc97-5f42-46fd-971a-aae237ea7657","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.167872026Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.167884124Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595,"session_id":"bd2dcc97-5f42-46fd-971a-aae237ea7657"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:28.167901611Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 1 len 12288 data: 2 3 3 02/30 Read block 57 len 24576 03/30 Write block 482 len 20480 data: 9 8 8 8 6 04/30 Write block 192 len 32768 data: 4 4 5 5 7 7 7 9 05/30 Read block 105 len 4096 06/30 Read block 377 len 20480 07/30 Flush 08/30 Write block 52 len 36864 data: 12 9 10 7 9 11 12 12 11 09/30 Write block 581 len 8192 data: 9 10 10/30 Write block 167 len 4096 data: 10 11/30 Write block 385 len 12288 data: 4 5 8 12/30 Read block 546 len 36864 13/30 Read block 109 len 20480 14/30 Read block 92 len 12288 15/30 Flush 16/30 Write block 503 len 36864 data: 9 9 8 8 8 8 7 6 6 17/30 Write block 179 len 36864 data: 11 8 8 9 11 12 12 13 12 18/30 Write block 483 len 28672 data: 9 9 9 7 6 4 5 19/30 Write block 589 len 28672 data: 11 9 9 9 9 9 8 20/30 Read block 571 len 12288 21/30 Write block 589 len 24576 data: 12 10 10 10 10 10 22/30 Read block 100 len 40960 23/30 Read block 294 len 12288 24/30 Read block 168 len 40960 25/30 Write block 313 len 32768 data: 10 10 10 10 11 12 10 9 26/30 Read block 284 len 24576 27/30 Read block 321 len 8192 28/30 Write block 222 len 4096 data: 7 29/30 Read block 531 len 8192 30/30 Read block 229 len 28672 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 9 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 7 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 15 14 15 44 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:34:28.18715677Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1595} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 152 152 152 F F F 1 020-039 99 99 99 149 149 149 F F F 2 040-059 102 102 102 153 153 153 F F F 3 060-079 102 102 102 153 153 153 F F F 4 080-099 101 101 101 151 151 151 F F F 5 100-119 101 101 101 151 151 151 F F F 6 120-139 96 96 96 143 143 143 F F F 7 140-159 96 96 96 144 144 144 F F F 8 160-179 102 102 102 153 153 153 T T T 9 180-199 102 102 102 152 152 152 T T T 10 200-219 101 101 101 151 151 151 F F F 11 220-239 101 101 101 151 151 151 T T T 12 240-259 101 101 101 151 151 151 F F F 13 260-279 101 101 101 151 151 151 F F F 14 280-299 96 96 96 144 144 144 F F F 15 300-319 101 101 101 151 151 151 T T T 16 320-339 101 101 101 151 151 151 T T T 17 340-359 100 100 100 150 150 150 F F F 18 360-379 98 98 98 148 148 148 F F F 19 380-399 102 102 102 153 153 153 F F F 20 400-419 101 101 101 151 151 151 F F F 21 420-439 99 99 99 149 149 149 F F F 22 440-459 98 98 98 147 147 147 F F F 23 460-479 99 99 99 149 149 149 F F F 24 480-499 102 102 102 152 152 152 T T T 25 500-519 99 99 99 149 149 149 T T T 26 520-539 97 97 97 145 145 145 F F F 27 540-559 95 95 95 142 142 142 F F F 28 560-579 101 101 101 151 151 151 F F F 29 580-599 102 102 102 153 153 153 T T T Max gen: 102, Max flush: 153 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 152 152 152 F F F 1 020-039 99 99 99 149 149 149 F F F 2 040-059 102 102 102 153 153 153 F F F 3 060-079 102 102 102 153 153 153 F F F 4 080-099 101 101 101 151 151 151 F F F 5 100-119 101 101 101 151 151 151 F F F 6 120-139 96 96 96 143 143 143 F F F 7 140-159 96 96 96 144 144 144 F F F 8 160-179 103 103 103 154 154 154 F F F 9 180-199 103 103 103 154 154 154 F F F 10 200-219 101 101 101 151 151 151 F F F 11 220-239 103 103 103 154 154 154 F F F 12 240-259 101 101 101 151 151 151 F F F 13 260-279 101 101 101 151 151 151 F F F 14 280-299 96 96 96 144 144 144 F F F 15 300-319 103 103 103 154 154 154 F F F 16 320-339 103 103 103 154 154 154 F F F 17 340-359 100 100 100 150 150 150 F F F 18 360-379 98 98 98 148 148 148 F F F 19 380-399 102 102 102 153 153 153 F F F 20 400-419 101 101 101 151 151 151 F F F 21 420-439 99 99 99 149 149 149 F F F 22 440-459 98 98 98 147 147 147 F F F 23 460-479 99 99 99 149 149 149 F F F 24 480-499 103 103 103 154 154 154 F F F 25 500-519 103 103 103 154 154 154 F F F 26 520-539 97 97 97 145 145 145 F F F 27 540-559 95 95 95 142 142 142 F F F 28 560-579 101 101 101 151 151 151 F F F 29 580-599 103 103 103 154 154 154 F F F Max gen: 103, Max flush: 154 Begin loop 52 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.471293136Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:34:32.522651337Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.523138196Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.555189756Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.555763822Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.555797688Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.555811906Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.555826084Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603} {"msg":"Crucible aff611f9-1dfe-4c88-b126-aa9ffa1af8c4 has session id: 86e9c009-842c-4817-a629-1a8eb70efea9","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.556273878Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"Upstairs opts: Upstairs UUID: aff611f9-1dfe-4c88-b126-aa9ffa1af8c4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.556319272Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"Crucible stats registered with UUID: aff611f9-1dfe-4c88-b126-aa9ffa1af8c4","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.556337469Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.587316615Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.587357629Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603} {"msg":"aff611f9-1dfe-4c88-b126-aa9ffa1af8c4 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.587387545Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.587406162Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"client":"0","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.587445047Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"client":"0","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.587481191Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"client":"1","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.587517336Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"client":"1","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.587551891Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"client":"2","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.587585776Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"client":"2","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.587620691Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.587658045Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.587781147Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"ds_connection connected from Ok(127.0.0.1:55531)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.587827111Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"ds_connection connected from Ok(127.0.0.1:61415)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.587858786Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"ds_connection connected from Ok(127.0.0.1:52948)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.587889482Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.588435232Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"client":"2","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.588469697Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"client":"2","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.588513871Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"client":"1","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.588546156Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"client":"1","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.588574312Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"client":"0","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.588604477Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"client":"0","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.589109404Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"client":"1","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.589224717Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"client":"0","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.589284338Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"client":"2","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.589660133Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"client":"1","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.589701887Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"client":"1","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.589719375Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.589857424Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"client":"0","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.58988764Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"client":"0","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.589904357Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.589917376Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"client":"2","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.589932244Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"client":"2","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"[0]R flush_numbers[0..12]: [152, 149, 153, 153, 151, 151, 143, 144, 154, 154, 151, 154]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.589946851Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"":"downstairs","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"[0]R generation[0..12]: [102, 99, 102, 102, 101, 101, 96, 96, 103, 103, 101, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.589964359Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"":"downstairs","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.589979107Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"":"downstairs","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"[1]R flush_numbers[0..12]: [152, 149, 153, 153, 151, 151, 143, 144, 154, 154, 151, 154]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.589996494Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"":"downstairs","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"[1]R generation[0..12]: [102, 99, 102, 102, 101, 101, 96, 96, 103, 103, 101, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.590020831Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"":"downstairs","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.590037638Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"":"downstairs","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"[2]R flush_numbers[0..12]: [152, 149, 153, 153, 151, 151, 143, 144, 154, 154, 151, 154]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.590056155Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"":"downstairs","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"[2]R generation[0..12]: [102, 99, 102, 102, 101, 101, 96, 96, 103, 103, 101, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.590073213Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"":"downstairs","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.590087871Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"":"downstairs","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.590110967Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"":"downstairs","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.590124316Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"":"downstairs","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"Next flush: 155","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.590137474Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"":"downstairs","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.590150482Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"":"downstairs","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.59016389Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.590176068Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"client":"0","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.590191855Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"client":"1","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.590205404Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"client":"2","":"downstairs","session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.590218842Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.59023122Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"aff611f9-1dfe-4c88-b126-aa9ffa1af8c4 is now active with session: 86e9c009-842c-4817-a629-1a8eb70efea9","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.590244228Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.590256276Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603,"session_id":"86e9c009-842c-4817-a629-1a8eb70efea9"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:32.590268544Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 487 len 28672 data: 7 5 6 6 6 8 10 02/30 Read block 71 len 32768 03/30 Flush 04/30 Write block 419 len 28672 data: 10 7 8 5 4 5 7 05/30 Write block 375 len 12288 data: 8 8 8 06/30 Write block 91 len 12288 data: 8 7 7 07/30 Read block 330 len 16384 08/30 Write block 216 len 4096 data: 7 09/30 Flush 10/30 Write block 33 len 32768 data: 6 5 3 3 5 5 8 8 11/30 Flush 12/30 Write block 416 len 36864 data: 13 12 10 11 8 9 6 5 6 13/30 Read block 120 len 12288 14/30 Read block 33 len 40960 15/30 Write block 409 len 16384 data: 10 11 14 13 16/30 Write block 482 len 24576 data: 10 10 10 10 8 8 17/30 Read block 532 len 20480 18/30 Read block 588 len 16384 19/30 Write block 464 len 20480 data: 8 7 6 7 6 20/30 Flush 21/30 Flush 22/30 Write block 438 len 16384 data: 6 6 7 8 23/30 Write block 345 len 32768 data: 9 10 8 8 9 9 9 8 24/30 Write block 425 len 4096 data: 8 25/30 Read block 42 len 24576 26/30 Write block 244 len 4096 data: 13 27/30 Read block 319 len 20480 28/30 Read block 500 len 32768 29/30 Write block 60 len 32768 data: 12 9 11 13 14 12 12 12 30/30 Write block 145 len 36864 data: 11 11 12 9 10 7 5 5 4 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 4 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:34:32.609079768Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1603} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 152 152 152 F F F 1 020-039 104 104 104 157 157 157 F F F 2 040-059 104 104 104 157 157 157 F F F 3 060-079 102 102 102 153 153 153 T T T 4 080-099 104 104 104 156 156 156 F F F 5 100-119 101 101 101 151 151 151 F F F 6 120-139 96 96 96 143 143 143 F F F 7 140-159 96 96 96 144 144 144 T T T 8 160-179 103 103 103 154 154 154 F F F 9 180-199 103 103 103 154 154 154 F F F 10 200-219 104 104 104 156 156 156 F F F 11 220-239 103 103 103 154 154 154 F F F 12 240-259 101 101 101 151 151 151 T T T 13 260-279 101 101 101 151 151 151 F F F 14 280-299 96 96 96 144 144 144 F F F 15 300-319 103 103 103 154 154 154 F F F 16 320-339 103 103 103 154 154 154 F F F 17 340-359 100 100 100 150 150 150 T T T 18 360-379 104 104 104 156 156 156 F F F 19 380-399 102 102 102 153 153 153 F F F 20 400-419 104 104 104 158 158 158 F F F 21 420-439 104 104 104 158 158 158 T T T 22 440-459 98 98 98 147 147 147 T T T 23 460-479 104 104 104 158 158 158 F F F 24 480-499 104 104 104 158 158 158 F F F 25 500-519 103 103 103 154 154 154 F F F 26 520-539 97 97 97 145 145 145 F F F 27 540-559 95 95 95 142 142 142 F F F 28 560-579 101 101 101 151 151 151 F F F 29 580-599 103 103 103 154 154 154 F F F Max gen: 104, Max flush: 158 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 152 152 152 F F F 1 020-039 104 104 104 157 157 157 F F F 2 040-059 104 104 104 157 157 157 F F F 3 060-079 105 105 105 159 159 159 F F F 4 080-099 104 104 104 156 156 156 F F F 5 100-119 101 101 101 151 151 151 F F F 6 120-139 96 96 96 143 143 143 F F F 7 140-159 105 105 105 159 159 159 F F F 8 160-179 103 103 103 154 154 154 F F F 9 180-199 103 103 103 154 154 154 F F F 10 200-219 104 104 104 156 156 156 F F F 11 220-239 103 103 103 154 154 154 F F F 12 240-259 105 105 105 159 159 159 F F F 13 260-279 101 101 101 151 151 151 F F F 14 280-299 96 96 96 144 144 144 F F F 15 300-319 103 103 103 154 154 154 F F F 16 320-339 103 103 103 154 154 154 F F F 17 340-359 105 105 105 159 159 159 F F F 18 360-379 104 104 104 156 156 156 F F F 19 380-399 102 102 102 153 153 153 F F F 20 400-419 104 104 104 158 158 158 F F F 21 420-439 105 105 105 159 159 159 F F F 22 440-459 105 105 105 159 159 159 F F F 23 460-479 104 104 104 158 158 158 F F F 24 480-499 104 104 104 158 158 158 F F F 25 500-519 103 103 103 154 154 154 F F F 26 520-539 97 97 97 145 145 145 F F F 27 540-559 95 95 95 142 142 142 F F F 28 560-579 101 101 101 151 151 151 F F F 29 580-599 103 103 103 154 154 154 F F F Max gen: 105, Max flush: 159 Begin loop 53 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:36.922443535Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:36.967143479Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:36.967762708Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:36.967799063Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:36.967813731Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:36.967828638Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611} {"msg":"Crucible 0b9fb449-2908-42c6-928d-bc703d4a325c has session id: 4ccd9229-5541-4d9e-a70d-0114984d297d","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:36.968264885Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"Upstairs opts: Upstairs UUID: 0b9fb449-2908-42c6-928d-bc703d4a325c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:36.968307679Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"Crucible stats registered with UUID: 0b9fb449-2908-42c6-928d-bc703d4a325c","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:36.968332605Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:36.999077076Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:36.999151305Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611} {"msg":"0b9fb449-2908-42c6-928d-bc703d4a325c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:36.999177901Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:36.999193029Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"client":"0","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:36.999216086Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"client":"0","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:36.999234533Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"client":"1","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:36.99925399Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"client":"1","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:36.999279586Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"client":"2","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:36.999306113Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"client":"2","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:36.999325939Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"":"io task","client":"2","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:36.999351436Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"":"io task","client":"0","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:36.999379172Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"ds_connection connected from Ok(127.0.0.1:57901)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:36.99945918Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"":"io task","client":"2","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"ds_connection connected from Ok(127.0.0.1:63223)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:36.999654962Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"":"io task","client":"0","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"ds_connection connected from Ok(127.0.0.1:40047)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:36.999683627Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.000224158Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"client":"0","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.000288129Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"client":"0","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.000321904Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"client":"2","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.00034902Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"client":"2","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.000885182Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"client":"1","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.000919957Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"client":"1","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.000947923Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"client":"2","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.00096386Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"client":"0","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.00151632Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"client":"1","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.001637952Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"client":"0","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.001657869Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"client":"0","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.001671717Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.001684355Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"client":"2","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.001697983Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"client":"2","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.001711171Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.002011797Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"client":"1","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.002036484Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"client":"1","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"[0]R flush_numbers[0..12]: [152, 157, 157, 159, 156, 151, 143, 159, 154, 154, 156, 154]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.002051212Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"":"downstairs","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"[0]R generation[0..12]: [102, 104, 104, 105, 104, 101, 96, 105, 103, 103, 104, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.00206537Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"":"downstairs","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.002080007Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"":"downstairs","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"[1]R flush_numbers[0..12]: [152, 157, 157, 159, 156, 151, 143, 159, 154, 154, 156, 154]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.002097555Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"":"downstairs","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"[1]R generation[0..12]: [102, 104, 104, 105, 104, 101, 96, 105, 103, 103, 104, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.002111833Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"":"downstairs","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.002125921Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"":"downstairs","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"[2]R flush_numbers[0..12]: [152, 157, 157, 159, 156, 151, 143, 159, 154, 154, 156, 154]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.002139799Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"":"downstairs","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"[2]R generation[0..12]: [102, 104, 104, 105, 104, 101, 96, 105, 103, 103, 104, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.002153567Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"":"downstairs","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.002167795Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"":"downstairs","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.002181023Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"":"downstairs","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.002194421Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"":"downstairs","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"Next flush: 160","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.002207619Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"":"downstairs","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:34:37.002220767Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"":"downstairs","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.002288157Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.002305015Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"client":"0","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.002318653Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"client":"1","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.002332001Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"client":"2","":"downstairs","session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.002345479Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.002361496Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"0b9fb449-2908-42c6-928d-bc703d4a325c is now active with session: 4ccd9229-5541-4d9e-a70d-0114984d297d","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.002374944Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.002387192Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611,"session_id":"4ccd9229-5541-4d9e-a70d-0114984d297d"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:37.002399871Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 288 len 8192 data: 7 8 02/30 Read block 400 len 36864 03/30 Read block 538 len 12288 04/30 Flush 05/30 Write block 457 len 4096 data: 12 06/30 Write block 546 len 36864 data: 9 10 13 10 12 13 14 12 10 07/30 Read block 511 len 40960 08/30 Write block 378 len 20480 data: 7 7 6 4 5 09/30 Write block 99 len 32768 data: 16 13 12 14 13 11 9 11 10/30 Write block 256 len 12288 data: 11 11 11 11/30 Read block 518 len 28672 12/30 Write block 128 len 8192 data: 9 10 13/30 Read block 564 len 4096 14/30 Read block 321 len 40960 15/30 Read block 383 len 12288 16/30 Read block 362 len 4096 17/30 Read block 137 len 32768 18/30 Write block 164 len 20480 data: 12 11 10 11 7 19/30 Read block 197 len 8192 20/30 Write block 171 len 20480 data: 6 6 5 5 7 21/30 Write block 187 len 28672 data: 13 12 10 10 9 5 5 22/30 Flush 23/30 Write block 45 len 8192 data: 9 11 24/30 Write block 518 len 12288 data: 7 6 8 25/30 Read block 503 len 36864 26/30 Write block 445 len 32768 data: 11 11 10 10 10 12 11 9 27/30 Write block 544 len 24576 data: 8 8 10 11 14 11 28/30 Read block 480 len 40960 29/30 Write block 92 len 40960 data: 8 8 6 8 8 11 14 17 14 13 30/30 Read block 181 len 32768 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 2 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 10 Done Sent Done false 1029 Acked Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 0 3 Done 7 6 8 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:34:37.021985801Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1611} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 152 152 152 F F F 1 020-039 104 104 104 157 157 157 F F F 2 040-059 104 104 104 157 157 157 T T T 3 060-079 105 105 105 159 159 159 F F F 4 080-099 106 106 106 161 161 161 T T T 5 100-119 106 106 106 161 161 161 T T T 6 120-139 106 106 106 161 161 161 F F F 7 140-159 105 105 105 159 159 159 F F F 8 160-179 106 106 106 161 161 161 F F F 9 180-199 106 106 106 161 161 161 F F F 10 200-219 104 104 104 156 156 156 F F F 11 220-239 103 103 103 154 154 154 F F F 12 240-259 106 106 106 161 161 161 F F F 13 260-279 101 101 101 151 151 151 F F F 14 280-299 106 106 106 160 160 160 F F F 15 300-319 103 103 103 154 154 154 F F F 16 320-339 103 103 103 154 154 154 F F F 17 340-359 105 105 105 159 159 159 F F F 18 360-379 106 106 106 161 161 161 F F F 19 380-399 106 106 106 161 161 161 F F F 20 400-419 104 104 104 158 158 158 F F F 21 420-439 105 105 105 159 159 159 F F F 22 440-459 106 106 106 161 161 161 T T T 23 460-479 104 104 104 158 158 158 F F F 24 480-499 104 104 104 158 158 158 F F F 25 500-519 103 103 103 154 154 154 T T T 26 520-539 97 97 97 145 145 145 T T T 27 540-559 106 106 106 161 161 161 T T T 28 560-579 101 101 101 151 151 151 F F F 29 580-599 103 103 103 154 154 154 F F F Max gen: 106, Max flush: 161 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 152 152 152 F F F 1 020-039 104 104 104 157 157 157 F F F 2 040-059 107 107 107 162 162 162 F F F 3 060-079 105 105 105 159 159 159 F F F 4 080-099 107 107 107 162 162 162 F F F 5 100-119 107 107 107 162 162 162 F F F 6 120-139 106 106 106 161 161 161 F F F 7 140-159 105 105 105 159 159 159 F F F 8 160-179 106 106 106 161 161 161 F F F 9 180-199 106 106 106 161 161 161 F F F 10 200-219 104 104 104 156 156 156 F F F 11 220-239 103 103 103 154 154 154 F F F 12 240-259 106 106 106 161 161 161 F F F 13 260-279 101 101 101 151 151 151 F F F 14 280-299 106 106 106 160 160 160 F F F 15 300-319 103 103 103 154 154 154 F F F 16 320-339 103 103 103 154 154 154 F F F 17 340-359 105 105 105 159 159 159 F F F 18 360-379 106 106 106 161 161 161 F F F 19 380-399 106 106 106 161 161 161 F F F 20 400-419 104 104 104 158 158 158 F F F 21 420-439 105 105 105 159 159 159 F F F 22 440-459 107 107 107 162 162 162 F F F 23 460-479 104 104 104 158 158 158 F F F 24 480-499 104 104 104 158 158 158 F F F 25 500-519 107 107 107 162 162 162 F F F 26 520-539 107 107 107 162 162 162 F F F 27 540-559 107 107 107 162 162 162 F F F 28 560-579 101 101 101 151 151 151 F F F 29 580-599 103 103 103 154 154 154 F F F Max gen: 107, Max flush: 162 Begin loop 54 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.302711005Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.350527374Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.351136515Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.35117126Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.351185448Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.351200645Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619} {"msg":"Crucible 08fd611f-bff7-4e21-b8b7-b7114c78f42f has session id: 97a36d4e-f321-43a0-9f90-08589f9043b7","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.351609886Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"Upstairs opts: Upstairs UUID: 08fd611f-bff7-4e21-b8b7-b7114c78f42f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.35164805Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"Crucible stats registered with UUID: 08fd611f-bff7-4e21-b8b7-b7114c78f42f","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.351665577Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.381813466Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.381844681Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619} {"msg":"08fd611f-bff7-4e21-b8b7-b7114c78f42f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.381892254Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.381912911Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"client":"0","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.381940967Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"client":"0","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.381960564Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"client":"1","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.381979291Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"client":"1","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.382005178Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"client":"2","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.382041822Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"client":"2","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.382074348Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"":"io task","client":"0","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.382108432Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"":"io task","client":"2","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.382142018Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:36005)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.382175833Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"":"io task","client":"2","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:34359)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.382208418Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:45416)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.382360016Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"":"io task","client":"0","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.382885089Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"client":"2","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.382913185Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"client":"2","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.382930303Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"client":"0","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.38294976Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"client":"0","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.383021089Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"client":"1","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.383051635Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"client":"1","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.383551002Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"client":"0","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.383638279Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"client":"2","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.383731935Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"client":"1","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.3841797Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"client":"0","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.384220734Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"client":"0","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.38425239Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.38431517Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"client":"1","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.384344436Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"client":"1","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.384359344Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.384371872Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"client":"2","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.38438587Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"client":"2","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"[0]R flush_numbers[0..12]: [152, 157, 162, 159, 162, 162, 161, 159, 161, 161, 156, 154]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.384400038Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"":"downstairs","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"[0]R generation[0..12]: [102, 104, 107, 105, 107, 107, 106, 105, 106, 106, 104, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.384414176Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"":"downstairs","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.384428694Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"":"downstairs","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"[1]R flush_numbers[0..12]: [152, 157, 162, 159, 162, 162, 161, 159, 161, 161, 156, 154]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.384442852Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"":"downstairs","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"[1]R generation[0..12]: [102, 104, 107, 105, 107, 107, 106, 105, 106, 106, 104, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.38445688Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"":"downstairs","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.384470998Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"":"downstairs","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"[2]R flush_numbers[0..12]: [152, 157, 162, 159, 162, 162, 161, 159, 161, 161, 156, 154]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.384488155Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"":"downstairs","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"[2]R generation[0..12]: [102, 104, 107, 105, 107, 107, 106, 105, 106, 106, 104, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.384502573Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"":"downstairs","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.384516711Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"":"downstairs","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:34:41.384530099Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"":"downstairs","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.384548226Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"":"downstairs","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"Next flush: 163","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.384561204Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"":"downstairs","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.384574332Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"":"downstairs","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.38458781Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.384600359Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"client":"0","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.384613667Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"client":"1","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.384626855Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"client":"2","":"downstairs","session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.384640373Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.384652711Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"08fd611f-bff7-4e21-b8b7-b7114c78f42f is now active with session: 97a36d4e-f321-43a0-9f90-08589f9043b7","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.384665929Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.384678217Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619,"session_id":"97a36d4e-f321-43a0-9f90-08589f9043b7"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:41.384693785Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 466 len 32768 data: 7 8 7 9 9 9 9 10 02/30 Read block 38 len 36864 03/30 Read block 54 len 8192 04/30 Flush 05/30 Write block 182 len 12288 data: 10 12 13 06/30 Read block 574 len 32768 07/30 Write block 274 len 28672 data: 8 8 7 7 5 3 4 08/30 Read block 159 len 8192 09/30 Read block 100 len 32768 10/30 Write block 302 len 4096 data: 7 11/30 Read block 65 len 4096 12/30 Write block 290 len 4096 data: 6 13/30 Read block 35 len 24576 14/30 Read block 114 len 8192 15/30 Write block 419 len 12288 data: 12 9 10 16/30 Read block 390 len 16384 17/30 Read block 278 len 32768 18/30 Write block 459 len 20480 data: 11 11 10 10 7 19/30 Write block 489 len 12288 data: 7 7 7 20/30 Write block 63 len 24576 data: 14 15 13 13 13 12 21/30 Write block 317 len 32768 data: 12 13 11 10 8 7 8 8 22/30 Write block 455 len 32768 data: 12 9 13 13 12 12 11 11 23/30 Write block 359 len 36864 data: 7 7 8 6 7 7 9 8 8 24/30 Read block 97 len 20480 25/30 Read block 573 len 24576 26/30 Read block 110 len 12288 27/30 Read block 584 len 8192 28/30 Write block 461 len 16384 data: 12 12 8 9 29/30 Write block 392 len 40960 data: 13 11 11 9 9 10 11 11 11 10 30/30 Write block 583 len 4096 data: 11 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 3 Done Done Done false 1005 Acked Read 8 Done Done Done false 1006 Acked Write 7 Done Done Done false 1007 Acked Read 2 Done Done Done false 1008 Acked Read 8 Done Done Done false 1009 Acked Write 1 Done Done Done false 1010 Acked Read 1 Done Done Done false 1011 Acked Write 1 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Write 3 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 3 2 7 Done 24 23 24 71 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:34:41.402437574Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1619} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 152 152 152 F F F 1 020-039 104 104 104 157 157 157 F F F 2 040-059 107 107 107 162 162 162 F F F 3 060-079 105 105 105 159 159 159 T T T 4 080-099 107 107 107 162 162 162 F F F 5 100-119 107 107 107 162 162 162 F F F 6 120-139 106 106 106 161 161 161 F F F 7 140-159 105 105 105 159 159 159 F F F 8 160-179 106 106 106 161 161 161 F F F 9 180-199 106 106 106 161 161 161 T T T 10 200-219 104 104 104 156 156 156 F F F 11 220-239 103 103 103 154 154 154 F F F 12 240-259 106 106 106 161 161 161 F F F 13 260-279 101 101 101 151 151 151 T T T 14 280-299 106 106 106 160 160 160 T T T 15 300-319 103 103 103 154 154 154 T T T 16 320-339 103 103 103 154 154 154 T T T 17 340-359 105 105 105 159 159 159 T T T 18 360-379 106 106 106 161 161 161 T T T 19 380-399 106 106 106 161 161 161 T T T 20 400-419 104 104 104 158 158 158 T T T 21 420-439 105 105 105 159 159 159 T T T 22 440-459 107 107 107 162 162 162 T T T 23 460-479 108 108 108 163 163 163 T T T 24 480-499 104 104 104 158 158 158 T T T 25 500-519 107 107 107 162 162 162 F F F 26 520-539 107 107 107 162 162 162 F F F 27 540-559 107 107 107 162 162 162 F F F 28 560-579 101 101 101 151 151 151 F F F 29 580-599 103 103 103 154 154 154 T T T Max gen: 108, Max flush: 163 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 152 152 152 F F F 1 020-039 104 104 104 157 157 157 F F F 2 040-059 107 107 107 162 162 162 F F F 3 060-079 109 109 109 164 164 164 F F F 4 080-099 107 107 107 162 162 162 F F F 5 100-119 107 107 107 162 162 162 F F F 6 120-139 106 106 106 161 161 161 F F F 7 140-159 105 105 105 159 159 159 F F F 8 160-179 106 106 106 161 161 161 F F F 9 180-199 109 109 109 164 164 164 F F F 10 200-219 104 104 104 156 156 156 F F F 11 220-239 103 103 103 154 154 154 F F F 12 240-259 106 106 106 161 161 161 F F F 13 260-279 109 109 109 164 164 164 F F F 14 280-299 109 109 109 164 164 164 F F F 15 300-319 109 109 109 164 164 164 F F F 16 320-339 109 109 109 164 164 164 F F F 17 340-359 109 109 109 164 164 164 F F F 18 360-379 109 109 109 164 164 164 F F F 19 380-399 109 109 109 164 164 164 F F F 20 400-419 109 109 109 164 164 164 F F F 21 420-439 109 109 109 164 164 164 F F F 22 440-459 109 109 109 164 164 164 F F F 23 460-479 109 109 109 164 164 164 F F F 24 480-499 109 109 109 164 164 164 F F F 25 500-519 107 107 107 162 162 162 F F F 26 520-539 107 107 107 162 162 162 F F F 27 540-559 107 107 107 162 162 162 F F F 28 560-579 101 101 101 151 151 151 F F F 29 580-599 109 109 109 164 164 164 F F F Max gen: 109, Max flush: 164 Begin loop 55 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.728523085Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.772312221Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.773015618Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.77306674Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.773088947Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.773113594Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627} {"msg":"Crucible ec4239c8-f425-44cc-8df7-144df679ac73 has session id: ebe159a9-2f4c-48aa-a6f1-21fea8c60b38","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.773531393Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"Upstairs opts: Upstairs UUID: ec4239c8-f425-44cc-8df7-144df679ac73, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.773582145Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"Crucible stats registered with UUID: ec4239c8-f425-44cc-8df7-144df679ac73","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.773608731Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.80464434Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.804678885Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627} {"msg":"ec4239c8-f425-44cc-8df7-144df679ac73 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.804702881Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.804720328Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"client":"0","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.804748395Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"client":"0","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.804766322Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"client":"1","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.804783279Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"client":"1","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.804804576Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"client":"2","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.804822264Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"client":"2","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.804838871Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"":"io task","client":"0","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.804855909Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"":"io task","client":"2","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.804872876Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"":"io task","client":"1","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"ds_connection connected from Ok(127.0.0.1:63110)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.804977561Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"":"io task","client":"0","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"ds_connection connected from Ok(127.0.0.1:64039)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.805020195Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"":"io task","client":"1","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"ds_connection connected from Ok(127.0.0.1:39300)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.805221005Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"":"io task","client":"2","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.805763536Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"client":"0","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.805795551Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"client":"0","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.805810949Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"client":"1","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.80587345Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"client":"1","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.805915704Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"client":"2","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.805949209Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"client":"2","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.806407992Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"client":"1","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.806454885Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"client":"0","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.80649157Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"client":"2","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.807018593Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"client":"0","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.807059187Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"client":"0","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.807089333Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.807112799Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"client":"2","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.807136866Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"client":"2","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.807161402Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.807184819Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"client":"1","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.807209415Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"client":"1","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"[0]R flush_numbers[0..12]: [152, 157, 162, 164, 162, 162, 161, 159, 161, 164, 156, 154]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.807232822Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"":"downstairs","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"[0]R generation[0..12]: [102, 104, 107, 109, 107, 107, 106, 105, 106, 109, 104, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.807258628Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"":"downstairs","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.807286664Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"":"downstairs","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"[1]R flush_numbers[0..12]: [152, 157, 162, 164, 162, 162, 161, 159, 161, 164, 156, 154]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.8073126Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"":"downstairs","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"[1]R generation[0..12]: [102, 104, 107, 109, 107, 107, 106, 105, 106, 109, 104, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.807339426Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"":"downstairs","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:34:45.807373141Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"":"downstairs","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"[2]R flush_numbers[0..12]: [152, 157, 162, 164, 162, 162, 161, 159, 161, 164, 156, 154]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.807413115Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"":"downstairs","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"[2]R generation[0..12]: [102, 104, 107, 109, 107, 107, 106, 105, 106, 109, 104, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.807444121Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"":"downstairs","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.807474496Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"":"downstairs","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.807502832Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"":"downstairs","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.807531188Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"":"downstairs","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"Next flush: 165","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2025-01-03T08:34:45.807559534Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"":"downstairs","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.807592649Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"":"downstairs","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.807608807Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.807621135Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"client":"0","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.807634513Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"client":"1","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.807647541Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"client":"2","":"downstairs","session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.807660949Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.807673367Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"ec4239c8-f425-44cc-8df7-144df679ac73 is now active with session: ebe159a9-2f4c-48aa-a6f1-21fea8c60b38","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.807686475Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.807701753Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627,"session_id":"ebe159a9-2f4c-48aa-a6f1-21fea8c60b38"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:45.807714591Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 256 len 24576 data: 12 12 12 12 11 11 02/30 Read block 33 len 16384 03/30 Write block 192 len 28672 data: 6 6 6 6 8 8 8 04/30 Read block 61 len 32768 05/30 Read block 363 len 32768 06/30 Write block 410 len 4096 data: 12 07/30 Read block 184 len 32768 08/30 Write block 141 len 40960 data: 8 9 10 10 12 12 13 10 11 8 09/30 Read block 278 len 36864 10/30 Read block 283 len 20480 11/30 Write block 292 len 16384 data: 6 7 7 6 12/30 Write block 85 len 40960 data: 13 10 8 9 11 11 9 9 9 7 13/30 Write block 283 len 28672 data: 6 5 7 6 6 8 9 14/30 Write block 470 len 20480 data: 10 10 10 11 7 15/30 Read block 200 len 36864 16/30 Flush 17/30 Write block 445 len 32768 data: 12 12 11 11 11 13 12 10 18/30 Write block 407 len 40960 data: 9 10 11 13 15 14 14 17 15 14 19/30 Write block 377 len 8192 data: 9 8 20/30 Write block 570 len 32768 data: 7 7 8 8 3 3 4 5 21/30 Read block 564 len 40960 22/30 Read block 344 len 4096 23/30 Write block 266 len 16384 data: 11 10 11 9 24/30 Write block 154 len 28672 data: 5 6 8 7 8 9 10 25/30 Write block 540 len 28672 data: 4 6 9 9 9 9 11 26/30 Write block 422 len 24576 data: 7 6 7 9 6 7 27/30 Write block 31 len 4096 data: 7 28/30 Write block 294 len 16384 data: 8 7 8 8 29/30 Write block 36 len 28672 data: 4 6 6 9 9 8 11 30/30 Read block 268 len 12288 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 8 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 2 2 Done 14 14 12 40 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:34:45.826048724Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1627} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 152 152 152 F F F 1 020-039 104 104 104 157 157 157 T T T 2 040-059 107 107 107 162 162 162 T T T 3 060-079 109 109 109 164 164 164 F F F 4 080-099 110 110 110 165 165 165 F F F 5 100-119 107 107 107 162 162 162 F F F 6 120-139 106 106 106 161 161 161 F F F 7 140-159 110 110 110 165 165 165 T T T 8 160-179 106 106 106 161 161 161 T T T 9 180-199 110 110 110 165 165 165 F F F 10 200-219 104 104 104 156 156 156 F F F 11 220-239 103 103 103 154 154 154 F F F 12 240-259 110 110 110 165 165 165 F F F 13 260-279 110 110 110 165 165 165 T T T 14 280-299 110 110 110 165 165 165 T T T 15 300-319 109 109 109 164 164 164 F F F 16 320-339 109 109 109 164 164 164 F F F 17 340-359 109 109 109 164 164 164 F F F 18 360-379 109 109 109 164 164 164 T T T 19 380-399 109 109 109 164 164 164 F F F 20 400-419 110 110 110 165 165 165 T T T 21 420-439 109 109 109 164 164 164 T T T 22 440-459 109 109 109 164 164 164 T T T 23 460-479 110 110 110 165 165 165 F F F 24 480-499 109 109 109 164 164 164 F F F 25 500-519 107 107 107 162 162 162 F F F 26 520-539 107 107 107 162 162 162 F F F 27 540-559 107 107 107 162 162 162 T T T 28 560-579 101 101 101 151 151 151 T T T 29 580-599 109 109 109 164 164 164 F F F Max gen: 110, Max flush: 165 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 152 152 152 F F F 1 020-039 111 111 111 166 166 166 F F F 2 040-059 111 111 111 166 166 166 F F F 3 060-079 109 109 109 164 164 164 F F F 4 080-099 110 110 110 165 165 165 F F F 5 100-119 107 107 107 162 162 162 F F F 6 120-139 106 106 106 161 161 161 F F F 7 140-159 111 111 111 166 166 166 F F F 8 160-179 111 111 111 166 166 166 F F F 9 180-199 110 110 110 165 165 165 F F F 10 200-219 104 104 104 156 156 156 F F F 11 220-239 103 103 103 154 154 154 F F F 12 240-259 110 110 110 165 165 165 F F F 13 260-279 111 111 111 166 166 166 F F F 14 280-299 111 111 111 166 166 166 F F F 15 300-319 109 109 109 164 164 164 F F F 16 320-339 109 109 109 164 164 164 F F F 17 340-359 109 109 109 164 164 164 F F F 18 360-379 111 111 111 166 166 166 F F F 19 380-399 109 109 109 164 164 164 F F F 20 400-419 111 111 111 166 166 166 F F F 21 420-439 111 111 111 166 166 166 F F F 22 440-459 111 111 111 166 166 166 F F F 23 460-479 110 110 110 165 165 165 F F F 24 480-499 109 109 109 164 164 164 F F F 25 500-519 107 107 107 162 162 162 F F F 26 520-539 107 107 107 162 162 162 F F F 27 540-559 111 111 111 166 166 166 F F F 28 560-579 111 111 111 166 166 166 F F F 29 580-599 109 109 109 164 164 164 F F F Max gen: 111, Max flush: 166 Begin loop 56 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.13299421Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:34:50.175265628Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.175649972Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.206914637Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.207476665Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.20751239Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.207526508Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.207540835Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635} {"msg":"Crucible 503825b7-68b3-4575-918e-25a9c74a31d9 has session id: 362bba89-2faf-4eeb-9dba-321ae5e48238","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.20798716Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"Upstairs opts: Upstairs UUID: 503825b7-68b3-4575-918e-25a9c74a31d9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.208030784Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"Crucible stats registered with UUID: 503825b7-68b3-4575-918e-25a9c74a31d9","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.208048351Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.238371064Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.238420817Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635} {"msg":"503825b7-68b3-4575-918e-25a9c74a31d9 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.238437094Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.238453602Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"client":"0","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.238479538Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"client":"0","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.238497026Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"client":"1","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.238519932Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"client":"1","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.23853381Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"client":"2","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.238547518Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"client":"2","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.238560846Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"":"io task","client":"2","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.238574794Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"":"io task","client":"0","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.23859962Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"":"io task","client":"1","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"ds_connection connected from Ok(127.0.0.1:54279)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.238724302Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"":"io task","client":"2","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"ds_connection connected from Ok(127.0.0.1:41268)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.238777745Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"":"io task","client":"1","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"ds_connection connected from Ok(127.0.0.1:36186)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.238885169Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"":"io task","client":"0","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.239594395Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"client":"1","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.239621671Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"client":"1","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.239654466Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"client":"2","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.239682712Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"client":"2","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.239710358Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"client":"0","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.239738824Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"client":"0","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.240306841Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"client":"1","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.240351055Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"client":"2","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.24038171Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"client":"0","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.24093125Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"client":"1","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.240970694Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"client":"1","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.241018957Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.241044434Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"client":"0","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.2410688Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"client":"0","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.241093716Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.241117013Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"client":"2","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.24114038Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"client":"2","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"[0]R flush_numbers[0..12]: [152, 166, 166, 164, 165, 162, 161, 166, 166, 165, 156, 154]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.241163736Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"":"downstairs","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"[0]R generation[0..12]: [102, 111, 111, 109, 110, 107, 106, 111, 111, 110, 104, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.241188382Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"":"downstairs","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.241216229Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"":"downstairs","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"[1]R flush_numbers[0..12]: [152, 166, 166, 164, 165, 162, 161, 166, 166, 165, 156, 154]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.241242255Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"":"downstairs","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"[1]R generation[0..12]: [102, 111, 111, 109, 110, 107, 106, 111, 111, 110, 104, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.24127356Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"":"downstairs","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.241305136Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"Wait for a query_work_queue command to finish before sending IO ":"downstairs","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"[2]R flush_numbers[0..12]: [152, 166, 166, 164, 165, 162, 161, 166, 166, 165, 156, 154]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.24134571Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"":"downstairs","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"[2]R generation[0..12]: [102, 111, 111, 109, 110, 107, 106, 111, 111, 110, 104, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.241377415Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"":"downstairs","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.241407201Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"":"downstairs","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.241436086Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"":"downstairs","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.241465142Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"":"downstairs","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"Next flush: 167","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.241491378Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"":"downstairs","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.241517854Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"":"downstairs","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.24154419Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.241573836Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"client":"0","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.241596583Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"client":"1","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.241610581Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"client":"2","":"downstairs","session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.241624279Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.241636757Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"503825b7-68b3-4575-918e-25a9c74a31d9 is now active with session: 362bba89-2faf-4eeb-9dba-321ae5e48238","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.241652725Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.241665013Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635,"session_id":"362bba89-2faf-4eeb-9dba-321ae5e48238"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:50.241677441Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 108 len 20480 data: 10 9 5 5 5 02/30 Write block 226 len 20480 data: 7 7 9 8 9 03/30 Flush 04/30 Write block 435 len 36864 data: 8 8 5 7 7 8 9 10 10 05/30 Write block 467 len 16384 data: 9 8 10 11 06/30 Read block 157 len 8192 07/30 Write block 200 len 20480 data: 9 8 8 6 4 08/30 Write block 148 len 40960 data: 11 12 9 6 6 5 6 7 9 8 09/30 Read block 166 len 24576 10/30 Read block 469 len 16384 11/30 Read block 75 len 12288 12/30 Read block 55 len 8192 13/30 Write block 186 len 28672 data: 14 14 13 11 11 10 7 14/30 Flush 15/30 Write block 510 len 12288 data: 7 7 7 16/30 Read block 413 len 40960 17/30 Write block 456 len 8192 data: 10 14 18/30 Write block 27 len 32768 data: 7 8 8 9 8 4 7 6 19/30 Write block 9 len 32768 data: 9 9 8 10 10 10 12 12 20/30 Flush 21/30 Write block 452 len 12288 data: 11 10 11 22/30 Read block 248 len 8192 23/30 Write block 314 len 4096 data: 11 24/30 Write block 389 len 4096 data: 8 25/30 Write block 21 len 12288 data: 13 10 9 26/30 Read block 79 len 28672 27/30 Write block 476 len 4096 data: 10 28/30 Read block 91 len 12288 29/30 Write block 511 len 4096 data: 8 30/30 Write block 278 len 24576 data: 6 4 5 4 5 7 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 3 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 1 Sent Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 1 1 4 Done 8 9 9 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:34:50.259235867Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1635} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 169 169 169 F F F 1 020-039 112 112 112 169 169 169 T T T 2 040-059 111 111 111 166 166 166 F F F 3 060-079 109 109 109 164 164 164 F F F 4 080-099 110 110 110 165 165 165 F F F 5 100-119 112 112 112 167 167 167 F F F 6 120-139 106 106 106 161 161 161 F F F 7 140-159 112 112 112 168 168 168 F F F 8 160-179 111 111 111 166 166 166 F F F 9 180-199 112 112 112 168 168 168 F F F 10 200-219 112 112 112 168 168 168 F F F 11 220-239 112 112 112 167 167 167 F F F 12 240-259 110 110 110 165 165 165 F F F 13 260-279 111 111 111 166 166 166 T T T 14 280-299 111 111 111 166 166 166 T T T 15 300-319 109 109 109 164 164 164 T T T 16 320-339 109 109 109 164 164 164 F F F 17 340-359 109 109 109 164 164 164 F F F 18 360-379 111 111 111 166 166 166 F F F 19 380-399 109 109 109 164 164 164 T T T 20 400-419 111 111 111 166 166 166 F F F 21 420-439 112 112 112 168 168 168 F F F 22 440-459 112 112 112 169 169 169 T T T 23 460-479 112 112 112 168 168 168 T T T 24 480-499 109 109 109 164 164 164 F F F 25 500-519 112 112 112 169 169 169 T T T 26 520-539 107 107 107 162 162 162 F F F 27 540-559 111 111 111 166 166 166 F F F 28 560-579 111 111 111 166 166 166 F F F 29 580-599 109 109 109 164 164 164 F F F Max gen: 112, Max flush: 169 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 169 169 169 F F F 1 020-039 113 113 113 170 170 170 F F F 2 040-059 111 111 111 166 166 166 F F F 3 060-079 109 109 109 164 164 164 F F F 4 080-099 110 110 110 165 165 165 F F F 5 100-119 112 112 112 167 167 167 F F F 6 120-139 106 106 106 161 161 161 F F F 7 140-159 112 112 112 168 168 168 F F F 8 160-179 111 111 111 166 166 166 F F F 9 180-199 112 112 112 168 168 168 F F F 10 200-219 112 112 112 168 168 168 F F F 11 220-239 112 112 112 167 167 167 F F F 12 240-259 110 110 110 165 165 165 F F F 13 260-279 113 113 113 170 170 170 F F F 14 280-299 113 113 113 170 170 170 F F F 15 300-319 113 113 113 170 170 170 F F F 16 320-339 109 109 109 164 164 164 F F F 17 340-359 109 109 109 164 164 164 F F F 18 360-379 111 111 111 166 166 166 F F F 19 380-399 113 113 113 170 170 170 F F F 20 400-419 111 111 111 166 166 166 F F F 21 420-439 112 112 112 168 168 168 F F F 22 440-459 113 113 113 170 170 170 F F F 23 460-479 113 113 113 170 170 170 F F F 24 480-499 109 109 109 164 164 164 F F F 25 500-519 113 113 113 170 170 170 F F F 26 520-539 107 107 107 162 162 162 F F F 27 540-559 111 111 111 166 166 166 F F F 28 560-579 111 111 111 166 166 166 F F F 29 580-599 109 109 109 164 164 164 F F F Max gen: 113, Max flush: 170 Begin loop 57 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.571878231Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:34:54.61441009Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.614785415Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.646546628Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.647188674Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.647231128Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.647248465Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.647266413Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643} {"msg":"Crucible acb8f6d1-e1c0-4e5b-ba19-aeb02eda114e has session id: bb825a40-7cf5-42a3-b3f0-5604cf33c181","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.647776298Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"Upstairs opts: Upstairs UUID: acb8f6d1-e1c0-4e5b-ba19-aeb02eda114e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.64783198Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"Crucible stats registered with UUID: acb8f6d1-e1c0-4e5b-ba19-aeb02eda114e","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.647861276Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.678641671Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.678673827Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643} {"msg":"acb8f6d1-e1c0-4e5b-ba19-aeb02eda114e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.678706532Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.67872383Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"client":"0","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.678748596Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"client":"0","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.678766863Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"client":"1","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.678784361Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"client":"1","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.678806258Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"client":"2","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.678824315Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"client":"2","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.678846881Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"":"io task","client":"2","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.678878467Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"":"io task","client":"0","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.67892573Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"":"io task","client":"1","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"ds_connection connected from Ok(127.0.0.1:40662)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.678957016Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"":"io task","client":"2","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"ds_connection connected from Ok(127.0.0.1:48207)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.678983462Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"":"io task","client":"0","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"ds_connection connected from Ok(127.0.0.1:43703)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.679008998Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"":"io task","client":"1","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.679841336Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"client":"1","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.679880411Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"client":"1","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.679899488Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"client":"2","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.679914106Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"client":"2","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.680074282Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"client":"0","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.680120016Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"client":"0","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.680622552Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"client":"2","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.680655437Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"client":"0","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.680671465Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"client":"1","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.681180601Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"client":"2","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.681208767Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"client":"2","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.681223794Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.681295674Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"client":"1","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.681315741Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"client":"1","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.681329719Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.681342187Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"client":"0","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.681356255Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"client":"0","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"[0]R flush_numbers[0..12]: [169, 170, 166, 164, 165, 167, 161, 168, 166, 168, 168, 167]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.681370403Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"":"downstairs","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"[0]R generation[0..12]: [112, 113, 111, 109, 110, 112, 106, 112, 111, 112, 112, 112]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.681384561Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"":"downstairs","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.681398819Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"":"downstairs","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"[1]R flush_numbers[0..12]: [169, 170, 166, 164, 165, 167, 161, 168, 166, 168, 168, 167]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.681412757Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"":"downstairs","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"[1]R generation[0..12]: [112, 113, 111, 109, 110, 112, 106, 112, 111, 112, 112, 112]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.681426635Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"":"downstairs","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.681444102Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"":"downstairs","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"[2]R flush_numbers[0..12]: [169, 170, 166, 164, 165, 167, 161, 168, 166, 168, 168, 167]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.68145811Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"":"downstairs","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"[2]R generation[0..12]: [112, 113, 111, 109, 110, 112, 106, 112, 111, 112, 112, 112]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.681480997Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"":"downstairs","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.681508643Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"":"downstairs","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.681534989Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"":"downstairs","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"Wait for a query_work_queue command to finish before sending IO Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.681569744Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"":"downstairs","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"Next flush: 171","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.68159757Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"":"downstairs","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.681614428Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"":"downstairs","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.681639454Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.68166313Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"client":"0","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.681690686Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"client":"1","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.681707204Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"client":"2","":"downstairs","session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.681729301Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.681753347Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"acb8f6d1-e1c0-4e5b-ba19-aeb02eda114e is now active with session: bb825a40-7cf5-42a3-b3f0-5604cf33c181","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.681779973Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.681799261Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643,"session_id":"bb825a40-7cf5-42a3-b3f0-5604cf33c181"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:54.681815508Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 146 len 8192 data: 13 14 02/30 Read block 288 len 36864 03/30 Write block 90 len 20480 data: 12 10 10 10 8 04/30 Write block 95 len 28672 data: 9 9 12 15 18 15 14 05/30 Read block 538 len 8192 06/30 Write block 267 len 32768 data: 11 12 10 10 11 9 10 9 07/30 Write block 251 len 32768 data: 10 12 12 12 9 13 13 13 08/30 Write block 438 len 12288 data: 8 8 9 09/30 Read block 337 len 16384 10/30 Write block 64 len 28672 data: 16 14 14 14 13 10 9 11/30 Write block 227 len 36864 data: 8 10 9 10 8 9 10 11 11 12/30 Read block 217 len 40960 13/30 Read block 483 len 36864 14/30 Write block 419 len 28672 data: 13 10 11 8 7 8 10 15/30 Flush 16/30 Write block 487 len 36864 data: 9 6 8 8 8 9 11 10 11 17/30 Write block 247 len 36864 data: 13 12 10 11 11 13 13 13 10 18/30 Write block 384 len 24576 data: 5 5 6 9 6 9 19/30 Read block 482 len 32768 20/30 Write block 590 len 12288 data: 11 11 11 21/30 Write block 349 len 36864 data: 10 10 10 9 8 7 5 5 7 22/30 Write block 169 len 4096 data: 6 23/30 Write block 182 len 20480 data: 11 13 14 13 15 24/30 Flush 25/30 Write block 381 len 16384 data: 5 6 5 6 26/30 Write block 259 len 12288 data: 13 12 12 27/30 Read block 413 len 12288 28/30 Read block 182 len 40960 29/30 Read block 167 len 8192 30/30 Write block 565 len 40960 data: 7 8 8 9 9 8 8 9 9 4 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 4 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:34:54.701014655Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1643} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 169 169 169 F F F 1 020-039 113 113 113 170 170 170 F F F 2 040-059 111 111 111 166 166 166 F F F 3 060-079 114 114 114 171 171 171 F F F 4 080-099 114 114 114 171 171 171 F F F 5 100-119 114 114 114 171 171 171 F F F 6 120-139 106 106 106 161 161 161 F F F 7 140-159 114 114 114 171 171 171 F F F 8 160-179 114 114 114 172 172 172 F F F 9 180-199 114 114 114 172 172 172 F F F 10 200-219 112 112 112 168 168 168 F F F 11 220-239 114 114 114 171 171 171 F F F 12 240-259 114 114 114 172 172 172 T T T 13 260-279 114 114 114 171 171 171 T T T 14 280-299 113 113 113 170 170 170 F F F 15 300-319 113 113 113 170 170 170 F F F 16 320-339 109 109 109 164 164 164 F F F 17 340-359 114 114 114 172 172 172 F F F 18 360-379 111 111 111 166 166 166 F F F 19 380-399 114 114 114 172 172 172 T T T 20 400-419 114 114 114 171 171 171 F F F 21 420-439 114 114 114 171 171 171 F F F 22 440-459 114 114 114 171 171 171 F F F 23 460-479 113 113 113 170 170 170 F F F 24 480-499 114 114 114 172 172 172 F F F 25 500-519 113 113 113 170 170 170 F F F 26 520-539 107 107 107 162 162 162 F F F 27 540-559 111 111 111 166 166 166 F F F 28 560-579 111 111 111 166 166 166 T T T 29 580-599 114 114 114 172 172 172 F F F Max gen: 114, Max flush: 172 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 169 169 169 F F F 1 020-039 113 113 113 170 170 170 F F F 2 040-059 111 111 111 166 166 166 F F F 3 060-079 114 114 114 171 171 171 F F F 4 080-099 114 114 114 171 171 171 F F F 5 100-119 114 114 114 171 171 171 F F F 6 120-139 106 106 106 161 161 161 F F F 7 140-159 114 114 114 171 171 171 F F F 8 160-179 114 114 114 172 172 172 F F F 9 180-199 114 114 114 172 172 172 F F F 10 200-219 112 112 112 168 168 168 F F F 11 220-239 114 114 114 171 171 171 F F F 12 240-259 115 115 115 173 173 173 F F F 13 260-279 115 115 115 173 173 173 F F F 14 280-299 113 113 113 170 170 170 F F F 15 300-319 113 113 113 170 170 170 F F F 16 320-339 109 109 109 164 164 164 F F F 17 340-359 114 114 114 172 172 172 F F F 18 360-379 111 111 111 166 166 166 F F F 19 380-399 115 115 115 173 173 173 F F F 20 400-419 114 114 114 171 171 171 F F F 21 420-439 114 114 114 171 171 171 F F F 22 440-459 114 114 114 171 171 171 F F F 23 460-479 113 113 113 170 170 170 F F F 24 480-499 114 114 114 172 172 172 F F F 25 500-519 113 113 113 170 170 170 F F F 26 520-539 107 107 107 162 162 162 F F F 27 540-559 111 111 111 166 166 166 F F F 28 560-579 115 115 115 173 173 173 F F F 29 580-599 114 114 114 172 172 172 F F F Max gen: 115, Max flush: 173 Begin loop 58 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:58.994349547Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.037835048Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.038560472Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.038603575Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.038627652Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.038646879Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651} {"msg":"Crucible 42736892-44eb-46e2-a0e3-5ebfe6cf2f9a has session id: ade360e8-3a68-4e35-9fc8-50a4cd53668d","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.039125099Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"Upstairs opts: Upstairs UUID: 42736892-44eb-46e2-a0e3-5ebfe6cf2f9a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.039183521Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"Crucible stats registered with UUID: 42736892-44eb-46e2-a0e3-5ebfe6cf2f9a","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.039216246Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.069361344Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.069395719Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651} {"msg":"42736892-44eb-46e2-a0e3-5ebfe6cf2f9a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.069432094Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.069453741Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"client":"0","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.069497784Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"client":"0","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.069534309Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"client":"1","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.069570464Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"client":"1","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.069603309Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"client":"2","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.069639994Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"client":"2","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.069677288Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"":"io task","client":"2","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.069771125Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"":"io task","client":"0","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.069843694Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"":"io task","client":"1","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"ds_connection connected from Ok(127.0.0.1:51221)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.069923353Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"":"io task","client":"2","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"ds_connection connected from Ok(127.0.0.1:45274)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.069967736Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"":"io task","client":"0","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"ds_connection connected from Ok(127.0.0.1:59821)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.070003031Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"":"io task","client":"1","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.070289409Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"client":"2","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.070328833Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"client":"2","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.070472072Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"client":"0","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.070571418Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"client":"0","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.070863135Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"client":"1","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.070910488Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"client":"1","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.070949273Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"client":"2","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.071245769Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"client":"0","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.071597898Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"client":"1","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.071727839Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"client":"2","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.071756425Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"client":"2","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.071772272Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.07178549Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"client":"0","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.071799599Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"client":"0","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.071813286Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.072127321Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"client":"1","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.072148368Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"client":"1","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"[0]R flush_numbers[0..12]: [169, 170, 166, 171, 171, 171, 161, 171, 172, 172, 168, 171]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.072163925Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"":"downstairs","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"[0]R generation[0..12]: [112, 113, 111, 114, 114, 114, 106, 114, 114, 114, 112, 114]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.072178253Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"":"downstairs","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.072192481Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"":"downstairs","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"[1]R flush_numbers[0..12]: [169, 170, 166, 171, 171, 171, 161, 171, 172, 172, 168, 171]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.072206569Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"":"downstairs","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"[1]R generation[0..12]: [112, 113, 111, 114, 114, 114, 106, 114, 114, 114, 112, 114]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.072220427Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"":"downstairs","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.072234375Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"":"downstairs","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"[2]R flush_numbers[0..12]: [169, 170, 166, 171, 171, 171, 161, 171, 172, 172, 168, 171]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.072248393Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"":"downstairs","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"[2]R generation[0..12]: [112, 113, 111, 114, 114, 114, 106, 114, 114, 114, 112, 114]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.07226488Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"":"downstairs","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.072279529Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"":"downstairs","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.072295116Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"":"downstairs","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.072311954Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"":"downstairs","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"Next flush: 174","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.072328081Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"":"downstairs","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-01-03T08:34:59.072344469Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"":"downstairs","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.072373055Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.072388362Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"client":"0","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.07240494Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"client":"1","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.072421768Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"client":"2","":"downstairs","session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.072438975Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.072454343Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"42736892-44eb-46e2-a0e3-5ebfe6cf2f9a is now active with session: ade360e8-3a68-4e35-9fc8-50a4cd53668d","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.07247047Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.072485648Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651,"session_id":"ade360e8-3a68-4e35-9fc8-50a4cd53668d"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:34:59.072501316Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 412 len 8192 data: 15 15 02/30 Read block 291 len 24576 03/30 Flush 04/30 Write block 565 len 36864 data: 8 9 9 10 10 9 9 10 10 05/30 Read block 305 len 28672 06/30 Read block 237 len 32768 07/30 Read block 414 len 16384 08/30 Write block 421 len 24576 data: 12 9 8 9 11 7 09/30 Write block 528 len 8192 data: 11 12 10/30 Read block 375 len 8192 11/30 Read block 582 len 28672 12/30 Write block 300 len 16384 data: 6 7 8 5 13/30 Write block 44 len 36864 data: 9 10 12 8 8 10 12 12 13 14/30 Read block 388 len 4096 15/30 Write block 129 len 12288 data: 11 11 10 16/30 Read block 119 len 20480 17/30 Read block 160 len 8192 18/30 Write block 550 len 8192 data: 13 14 19/30 Read block 415 len 8192 20/30 Flush 21/30 Write block 326 len 24576 data: 8 7 6 6 7 5 22/30 Read block 526 len 8192 23/30 Write block 492 len 4096 data: 10 24/30 Read block 380 len 8192 25/30 Read block 413 len 36864 26/30 Write block 183 len 36864 data: 14 15 14 16 15 14 12 12 11 27/30 Read block 272 len 32768 28/30 Read block 231 len 12288 29/30 Write block 129 len 36864 data: 12 12 11 11 9 8 8 6 7 30/30 Write block 164 len 32768 data: 13 12 11 12 8 7 5 7 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 6 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:34:59.091435141Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1651} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 169 169 169 F F F 1 020-039 113 113 113 170 170 170 F F F 2 040-059 116 116 116 175 175 175 F F F 3 060-079 114 114 114 171 171 171 F F F 4 080-099 114 114 114 171 171 171 F F F 5 100-119 114 114 114 171 171 171 F F F 6 120-139 116 116 116 175 175 175 T T T 7 140-159 114 114 114 171 171 171 F F F 8 160-179 114 114 114 172 172 172 T T T 9 180-199 114 114 114 172 172 172 T T T 10 200-219 112 112 112 168 168 168 F F F 11 220-239 114 114 114 171 171 171 F F F 12 240-259 115 115 115 173 173 173 F F F 13 260-279 115 115 115 173 173 173 F F F 14 280-299 113 113 113 170 170 170 F F F 15 300-319 116 116 116 175 175 175 F F F 16 320-339 109 109 109 164 164 164 T T T 17 340-359 114 114 114 172 172 172 F F F 18 360-379 111 111 111 166 166 166 F F F 19 380-399 115 115 115 173 173 173 F F F 20 400-419 116 116 116 174 174 174 F F F 21 420-439 116 116 116 175 175 175 F F F 22 440-459 114 114 114 171 171 171 F F F 23 460-479 113 113 113 170 170 170 F F F 24 480-499 114 114 114 172 172 172 T T T 25 500-519 113 113 113 170 170 170 F F F 26 520-539 116 116 116 175 175 175 F F F 27 540-559 116 116 116 175 175 175 F F F 28 560-579 116 116 116 175 175 175 F F F 29 580-599 114 114 114 172 172 172 F F F Max gen: 116, Max flush: 175 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 169 169 169 F F F 1 020-039 113 113 113 170 170 170 F F F 2 040-059 116 116 116 175 175 175 F F F 3 060-079 114 114 114 171 171 171 F F F 4 080-099 114 114 114 171 171 171 F F F 5 100-119 114 114 114 171 171 171 F F F 6 120-139 117 117 117 176 176 176 F F F 7 140-159 114 114 114 171 171 171 F F F 8 160-179 117 117 117 176 176 176 F F F 9 180-199 117 117 117 176 176 176 F F F 10 200-219 112 112 112 168 168 168 F F F 11 220-239 114 114 114 171 171 171 F F F 12 240-259 115 115 115 173 173 173 F F F 13 260-279 115 115 115 173 173 173 F F F 14 280-299 113 113 113 170 170 170 F F F 15 300-319 116 116 116 175 175 175 F F F 16 320-339 117 117 117 176 176 176 F F F 17 340-359 114 114 114 172 172 172 F F F 18 360-379 111 111 111 166 166 166 F F F 19 380-399 115 115 115 173 173 173 F F F 20 400-419 116 116 116 174 174 174 F F F 21 420-439 116 116 116 175 175 175 F F F 22 440-459 114 114 114 171 171 171 F F F 23 460-479 113 113 113 170 170 170 F F F 24 480-499 117 117 117 176 176 176 F F F 25 500-519 113 113 113 170 170 170 F F F 26 520-539 116 116 116 175 175 175 F F F 27 540-559 116 116 116 175 175 175 F F F 28 560-579 116 116 116 175 175 175 F F F 29 580-599 114 114 114 172 172 172 F F F Max gen: 117, Max flush: 176 Begin loop 59 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.358668586Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.402033444Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.402633586Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.402669851Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.402684789Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.402700927Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659} {"msg":"Crucible ae3a8a0d-a604-4bb4-9df6-8c3196c60979 has session id: 0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.403161939Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"Upstairs opts: Upstairs UUID: ae3a8a0d-a604-4bb4-9df6-8c3196c60979, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.403209013Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"Crucible stats registered with UUID: ae3a8a0d-a604-4bb4-9df6-8c3196c60979","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.403233659Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.433885113Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.433921898Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659} {"msg":"ae3a8a0d-a604-4bb4-9df6-8c3196c60979 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.433956513Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.43397721Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"client":"0","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.434018214Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"client":"0","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.434054069Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"client":"1","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.434088784Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"client":"1","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.434123148Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"client":"2","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.434157434Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"client":"2","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.434194718Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"":"io task","client":"2","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.434231903Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"":"io task","client":"0","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.434267937Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"":"io task","client":"1","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"ds_connection connected from Ok(127.0.0.1:39844)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.434310421Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"":"io task","client":"2","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"ds_connection connected from Ok(127.0.0.1:55507)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.434348316Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"":"io task","client":"1","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"ds_connection connected from Ok(127.0.0.1:38705)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.434482596Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"":"io task","client":"0","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.435037035Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"client":"0","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.435080039Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"client":"0","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.435113234Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"client":"1","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.43514409Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"client":"1","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.435173645Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"client":"2","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.435204781Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"client":"2","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.435711627Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"client":"0","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.435863984Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"client":"1","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.435901759Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"client":"2","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.43630716Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"client":"0","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.436335416Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"client":"0","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.436350303Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.436435721Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"client":"1","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.436464367Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"client":"1","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.436479584Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.436493362Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"client":"2","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.436507211Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"client":"2","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"[0]R flush_numbers[0..12]: [169, 170, 175, 171, 171, 171, 176, 171, 176, 176, 168, 171]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.436521408Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"":"downstairs","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"[0]R generation[0..12]: [112, 113, 116, 114, 114, 114, 117, 114, 117, 117, 112, 114]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.436536046Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"":"downstairs","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.436550504Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"":"downstairs","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"[1]R flush_numbers[0..12]: [169, 170, 175, 171, 171, 171, 176, 171, 176, 176, 168, 171]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.436564622Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"":"downstairs","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"[1]R generation[0..12]: [112, 113, 116, 114, 114, 114, 117, 114, 117, 117, 112, 114]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.43657855Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"":"downstairs","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.436592838Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"":"downstairs","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"[2]R flush_numbers[0..12]: [169, 170, 175, 171, 171, 171, 176, 171, 176, 176, 168, 171]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.436606836Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"":"downstairs","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"[2]R generation[0..12]: [112, 113, 116, 114, 114, 114, 117, 114, 117, 117, 112, 114]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.436620604Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"":"downstairs","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.436634532Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"":"downstairs","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.436652099Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"":"downstairs","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.436665987Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"":"downstairs","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"Next flush: 177","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:35:03.436679065Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"":"downstairs","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.436702932Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"":"downstairs","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.436721489Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.436739137Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"client":"0","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.436753375Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"client":"1","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.436766793Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"client":"2","":"downstairs","session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.43678054Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.436793119Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"ae3a8a0d-a604-4bb4-9df6-8c3196c60979 is now active with session: 0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.436806087Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.436818575Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659,"session_id":"0255b1ac-4cb1-4d09-ad4c-f7f508c3fe79"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:03.436830963Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 31 len 16384 data: 9 5 8 7 02/30 Read block 467 len 28672 03/30 Read block 211 len 4096 04/30 Flush 05/30 Write block 364 len 36864 data: 8 10 9 9 9 8 11 11 11 06/30 Read block 96 len 32768 07/30 Write block 224 len 28672 data: 9 6 8 9 11 10 11 08/30 Write block 477 len 20480 data: 9 7 7 9 9 09/30 Flush 10/30 Write block 202 len 4096 data: 9 11/30 Flush 12/30 Write block 436 len 40960 data: 9 6 9 9 10 10 11 11 11 13 13/30 Write block 347 len 8192 data: 9 9 14/30 Read block 337 len 8192 15/30 Flush 16/30 Write block 244 len 12288 data: 14 15 12 17/30 Write block 463 len 20480 data: 9 10 8 8 10 18/30 Read block 298 len 16384 19/30 Write block 231 len 32768 data: 9 10 11 12 12 11 14 13 20/30 Write block 203 len 28672 data: 7 5 4 5 6 5 5 21/30 Read block 173 len 12288 22/30 Write block 190 len 32768 data: 13 12 8 7 7 7 9 9 23/30 Read block 72 len 28672 24/30 Read block 231 len 8192 25/30 Write block 250 len 12288 data: 12 12 14 26/30 Read block 240 len 32768 27/30 Read block 338 len 36864 28/30 Write block 5 len 12288 data: 2 2 3 29/30 Write block 131 len 20480 data: 12 12 10 9 9 30/30 Write block 516 len 40960 data: 7 7 8 7 9 9 9 9 10 13 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 3 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:35:03.456256757Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1659} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 169 169 169 T T T 1 020-039 118 118 118 177 177 177 F F F 2 040-059 116 116 116 175 175 175 F F F 3 060-079 114 114 114 171 171 171 F F F 4 080-099 114 114 114 171 171 171 F F F 5 100-119 114 114 114 171 171 171 F F F 6 120-139 117 117 117 176 176 176 T T T 7 140-159 114 114 114 171 171 171 F F F 8 160-179 117 117 117 176 176 176 F F F 9 180-199 117 117 117 176 176 176 T T T 10 200-219 118 118 118 179 179 179 T T T 11 220-239 118 118 118 178 178 178 T T T 12 240-259 115 115 115 173 173 173 T T T 13 260-279 115 115 115 173 173 173 F F F 14 280-299 113 113 113 170 170 170 F F F 15 300-319 116 116 116 175 175 175 F F F 16 320-339 117 117 117 176 176 176 F F F 17 340-359 118 118 118 180 180 180 F F F 18 360-379 118 118 118 178 178 178 F F F 19 380-399 115 115 115 173 173 173 F F F 20 400-419 116 116 116 174 174 174 F F F 21 420-439 118 118 118 180 180 180 F F F 22 440-459 118 118 118 180 180 180 F F F 23 460-479 118 118 118 178 178 178 T T T 24 480-499 118 118 118 178 178 178 F F F 25 500-519 113 113 113 170 170 170 T T T 26 520-539 116 116 116 175 175 175 T T T 27 540-559 116 116 116 175 175 175 F F F 28 560-579 116 116 116 175 175 175 F F F 29 580-599 114 114 114 172 172 172 F F F Max gen: 118, Max flush: 180 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 181 181 181 F F F 1 020-039 118 118 118 177 177 177 F F F 2 040-059 116 116 116 175 175 175 F F F 3 060-079 114 114 114 171 171 171 F F F 4 080-099 114 114 114 171 171 171 F F F 5 100-119 114 114 114 171 171 171 F F F 6 120-139 119 119 119 181 181 181 F F F 7 140-159 114 114 114 171 171 171 F F F 8 160-179 117 117 117 176 176 176 F F F 9 180-199 119 119 119 181 181 181 F F F 10 200-219 119 119 119 181 181 181 F F F 11 220-239 119 119 119 181 181 181 F F F 12 240-259 119 119 119 181 181 181 F F F 13 260-279 115 115 115 173 173 173 F F F 14 280-299 113 113 113 170 170 170 F F F 15 300-319 116 116 116 175 175 175 F F F 16 320-339 117 117 117 176 176 176 F F F 17 340-359 118 118 118 180 180 180 F F F 18 360-379 118 118 118 178 178 178 F F F 19 380-399 115 115 115 173 173 173 F F F 20 400-419 116 116 116 174 174 174 F F F 21 420-439 118 118 118 180 180 180 F F F 22 440-459 118 118 118 180 180 180 F F F 23 460-479 119 119 119 181 181 181 F F F 24 480-499 118 118 118 178 178 178 F F F 25 500-519 119 119 119 181 181 181 F F F 26 520-539 119 119 119 181 181 181 F F F 27 540-559 116 116 116 175 175 175 F F F 28 560-579 116 116 116 175 175 175 F F F 29 580-599 114 114 114 172 172 172 F F F Max gen: 119, Max flush: 181 Begin loop 60 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.742706435Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.790228426Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.79088619Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.790923494Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.790938332Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.79095302Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667} {"msg":"Crucible e2e397ab-4b54-4cf7-a97a-cd30f7dadc8d has session id: d6850668-b91f-4df1-8677-4f81e7cbbf45","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.791434939Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"Upstairs opts: Upstairs UUID: e2e397ab-4b54-4cf7-a97a-cd30f7dadc8d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.791490861Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"Crucible stats registered with UUID: e2e397ab-4b54-4cf7-a97a-cd30f7dadc8d","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.791512418Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.822197138Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.822224374Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667} {"msg":"e2e397ab-4b54-4cf7-a97a-cd30f7dadc8d active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.822293114Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.82231671Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"client":"0","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.822344126Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"client":"0","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.822361924Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"client":"1","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.822379191Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"client":"1","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.822395959Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"client":"2","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.822412816Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"client":"2","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.822463479Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"":"io task","client":"2","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.822503093Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"":"io task","client":"0","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.822533978Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"":"io task","client":"1","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"ds_connection connected from Ok(127.0.0.1:35675)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.822561665Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"":"io task","client":"2","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"ds_connection connected from Ok(127.0.0.1:63092)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.82259282Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"":"io task","client":"1","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"ds_connection connected from Ok(127.0.0.1:42473)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.822627135Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"":"io task","client":"0","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.823366267Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"client":"1","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.823406631Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"client":"1","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.8235537Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"client":"0","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.823583915Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"client":"0","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.823599583Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"client":"2","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.823614031Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"client":"2","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.824038509Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"client":"1","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.824076943Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"client":"0","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.824325517Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"client":"2","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.824665617Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"client":"1","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.824706431Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"client":"1","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.824723419Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.824736467Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"client":"0","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.824750425Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"client":"0","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.824767532Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.82478669Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"client":"2","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.824801078Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"client":"2","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"[0]R flush_numbers[0..12]: [181, 177, 175, 171, 171, 171, 181, 171, 176, 181, 181, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.824830303Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"":"downstairs","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"[0]R generation[0..12]: [119, 118, 116, 114, 114, 114, 119, 114, 117, 119, 119, 119]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.824845281Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"":"downstairs","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.824859679Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"":"downstairs","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"[1]R flush_numbers[0..12]: [181, 177, 175, 171, 171, 171, 181, 171, 176, 181, 181, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.824874237Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"":"downstairs","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"[1]R generation[0..12]: [119, 118, 116, 114, 114, 114, 119, 114, 117, 119, 119, 119]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.824889345Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"":"downstairs","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.824903523Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"":"downstairs","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"[2]R flush_numbers[0..12]: [181, 177, 175, 171, 171, 171, 181, 171, 176, 181, 181, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.82491775Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"":"downstairs","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"[2]R generation[0..12]: [119, 118, 116, 114, 114, 114, 119, 114, 117, 119, 119, 119]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.824931928Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"":"downstairs","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.824946166Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"":"downstairs","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.824959535Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"":"downstairs","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.824972902Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"":"downstairs","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"Next flush: 182","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.82498842Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"":"downstairs","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:35:07.825002108Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"":"downstairs","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.825023455Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.825035873Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"client":"0","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.825049211Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"client":"1","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.825062469Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"client":"2","":"downstairs","session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.825075958Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.825088436Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"e2e397ab-4b54-4cf7-a97a-cd30f7dadc8d is now active with session: d6850668-b91f-4df1-8677-4f81e7cbbf45","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.825102323Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.825114742Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667,"session_id":"d6850668-b91f-4df1-8677-4f81e7cbbf45"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:07.82512722Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 354 len 16384 data: 8 6 6 8 02/30 Read block 64 len 36864 03/30 Write block 158 len 40960 data: 9 10 11 12 13 11 14 13 12 13 04/30 Write block 471 len 24576 data: 11 11 12 8 8 11 05/30 Read block 418 len 36864 06/30 Flush 07/30 Write block 240 len 12288 data: 13 12 11 08/30 Write block 559 len 12288 data: 6 5 5 09/30 Read block 377 len 16384 10/30 Read block 434 len 32768 11/30 Write block 350 len 32768 data: 11 11 10 9 9 7 7 9 12/30 Write block 226 len 32768 data: 9 10 12 11 12 10 11 12 13/30 Read block 502 len 16384 14/30 Read block 496 len 40960 15/30 Read block 181 len 16384 16/30 Read block 106 len 4096 17/30 Write block 228 len 36864 data: 13 12 13 11 12 13 13 13 12 18/30 Write block 513 len 8192 data: 8 9 19/30 Flush 20/30 Write block 233 len 16384 data: 14 14 14 13 21/30 Write block 581 len 40960 data: 10 11 12 12 9 10 12 11 13 12 22/30 Read block 198 len 24576 23/30 Flush 24/30 Write block 52 len 8192 data: 14 10 25/30 Read block 441 len 4096 26/30 Read block 155 len 40960 27/30 Write block 575 len 16384 data: 4 5 6 9 28/30 Read block 257 len 32768 29/30 Read block 257 len 8192 30/30 Read block 320 len 24576 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 2 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 6 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:35:07.844610565Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1667} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 181 181 181 F F F 1 020-039 118 118 118 177 177 177 F F F 2 040-059 116 116 116 175 175 175 T T T 3 060-079 114 114 114 171 171 171 F F F 4 080-099 114 114 114 171 171 171 F F F 5 100-119 114 114 114 171 171 171 F F F 6 120-139 119 119 119 181 181 181 F F F 7 140-159 120 120 120 182 182 182 F F F 8 160-179 120 120 120 182 182 182 F F F 9 180-199 119 119 119 181 181 181 F F F 10 200-219 119 119 119 181 181 181 F F F 11 220-239 120 120 120 184 184 184 F F F 12 240-259 120 120 120 183 183 183 F F F 13 260-279 115 115 115 173 173 173 F F F 14 280-299 113 113 113 170 170 170 F F F 15 300-319 116 116 116 175 175 175 F F F 16 320-339 117 117 117 176 176 176 F F F 17 340-359 120 120 120 183 183 183 F F F 18 360-379 118 118 118 178 178 178 F F F 19 380-399 115 115 115 173 173 173 F F F 20 400-419 116 116 116 174 174 174 F F F 21 420-439 118 118 118 180 180 180 F F F 22 440-459 118 118 118 180 180 180 F F F 23 460-479 120 120 120 182 182 182 F F F 24 480-499 118 118 118 178 178 178 F F F 25 500-519 120 120 120 183 183 183 F F F 26 520-539 119 119 119 181 181 181 F F F 27 540-559 120 120 120 183 183 183 F F F 28 560-579 120 120 120 183 183 183 T T T 29 580-599 120 120 120 184 184 184 F F F Max gen: 120, Max flush: 184 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 181 181 181 F F F 1 020-039 118 118 118 177 177 177 F F F 2 040-059 121 121 121 185 185 185 F F F 3 060-079 114 114 114 171 171 171 F F F 4 080-099 114 114 114 171 171 171 F F F 5 100-119 114 114 114 171 171 171 F F F 6 120-139 119 119 119 181 181 181 F F F 7 140-159 120 120 120 182 182 182 F F F 8 160-179 120 120 120 182 182 182 F F F 9 180-199 119 119 119 181 181 181 F F F 10 200-219 119 119 119 181 181 181 F F F 11 220-239 120 120 120 184 184 184 F F F 12 240-259 120 120 120 183 183 183 F F F 13 260-279 115 115 115 173 173 173 F F F 14 280-299 113 113 113 170 170 170 F F F 15 300-319 116 116 116 175 175 175 F F F 16 320-339 117 117 117 176 176 176 F F F 17 340-359 120 120 120 183 183 183 F F F 18 360-379 118 118 118 178 178 178 F F F 19 380-399 115 115 115 173 173 173 F F F 20 400-419 116 116 116 174 174 174 F F F 21 420-439 118 118 118 180 180 180 F F F 22 440-459 118 118 118 180 180 180 F F F 23 460-479 120 120 120 182 182 182 F F F 24 480-499 118 118 118 178 178 178 F F F 25 500-519 120 120 120 183 183 183 F F F 26 520-539 119 119 119 181 181 181 F F F 27 540-559 120 120 120 183 183 183 F F F 28 560-579 121 121 121 185 185 185 F F F 29 580-599 120 120 120 184 184 184 F F F Max gen: 121, Max flush: 185 Begin loop 61 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.096868347Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:35:12.139438941Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.139802708Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.171545803Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.17210958Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.172153884Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.172172011Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.172190309Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675} {"msg":"Crucible 78679192-4d48-422a-b391-7efd1059bdcc has session id: 9a82ddf0-74bf-4229-8192-dd8c0fef8ed0","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.172601299Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"Upstairs opts: Upstairs UUID: 78679192-4d48-422a-b391-7efd1059bdcc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.172646122Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"Crucible stats registered with UUID: 78679192-4d48-422a-b391-7efd1059bdcc","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.172677077Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.203349279Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.203380824Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675} {"msg":"78679192-4d48-422a-b391-7efd1059bdcc active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.203430147Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.203449894Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"client":"0","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.20347632Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"client":"0","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.203494048Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"client":"1","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.203511865Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"client":"1","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.203528773Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"client":"2","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.203558399Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"client":"2","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.203591803Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.203624999Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.203658994Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"ds_connection connected from Ok(127.0.0.1:36188)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.203796304Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"ds_connection connected from Ok(127.0.0.1:45317)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.203884251Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"ds_connection connected from Ok(127.0.0.1:46652)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.203923595Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.204514869Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"client":"1","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.204544524Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"client":"1","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.204745855Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"client":"0","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.204787799Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"client":"0","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.204819714Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"client":"2","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.2048485Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"client":"2","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.205186091Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"client":"1","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.205226895Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"client":"2","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.205616138Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"client":"0","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.205700076Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"client":"1","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.205729411Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"client":"1","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.205744629Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.205901696Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"client":"2","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.205921833Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"client":"2","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.205936531Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.206039206Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"client":"0","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.206063643Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"client":"0","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"[0]R flush_numbers[0..12]: [181, 177, 185, 171, 171, 171, 181, 182, 182, 181, 181, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.20607868Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"":"downstairs","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"[0]R generation[0..12]: [119, 118, 121, 114, 114, 114, 119, 120, 120, 119, 119, 120]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.206092798Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"":"downstairs","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.206107056Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"":"downstairs","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"[1]R flush_numbers[0..12]: [181, 177, 185, 171, 171, 171, 181, 182, 182, 181, 181, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.206121114Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"":"downstairs","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"[1]R generation[0..12]: [119, 118, 121, 114, 114, 114, 119, 120, 120, 119, 119, 120]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.206134992Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"":"downstairs","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.2061489Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"":"downstairs","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"[2]R flush_numbers[0..12]: [181, 177, 185, 171, 171, 171, 181, 182, 182, 181, 181, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.206162828Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"":"downstairs","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"[2]R generation[0..12]: [119, 118, 121, 114, 114, 114, 119, 120, 120, 119, 119, 120]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.206176636Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"":"downstairs","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.206191564Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"":"downstairs","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.206204992Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"":"downstairs","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.20621838Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"":"downstairs","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"Next flush: 186","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.206231668Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"":"downstairs","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"All extents match","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2025-01-03T08:35:12.206256754Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"":"downstairs","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.206270303Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.206282201Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"client":"0","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.206295569Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"client":"1","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.206308517Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"client":"2","":"downstairs","session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.206321845Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.206334323Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"78679192-4d48-422a-b391-7efd1059bdcc is now active with session: 9a82ddf0-74bf-4229-8192-dd8c0fef8ed0","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.206347361Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.206359499Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675,"session_id":"9a82ddf0-74bf-4229-8192-dd8c0fef8ed0"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:12.206371898Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 221 len 20480 data: 8 8 9 10 7 02/30 Read block 439 len 4096 03/30 Flush 04/30 Write block 322 len 24576 data: 8 9 9 8 9 8 05/30 Read block 573 len 32768 06/30 Write block 47 len 36864 data: 9 9 11 13 13 15 11 11 8 07/30 Read block 397 len 24576 08/30 Read block 430 len 32768 09/30 Write block 84 len 36864 data: 7 14 11 9 10 12 13 11 11 10/30 Read block 512 len 12288 11/30 Flush 12/30 Write block 113 len 28672 data: 6 7 7 8 11 13 14 13/30 Read block 460 len 32768 14/30 Flush 15/30 Write block 70 len 36864 data: 10 7 9 9 11 12 13 12 11 16/30 Write block 390 len 8192 data: 9 12 17/30 Read block 15 len 24576 18/30 Write block 111 len 36864 data: 6 6 7 8 8 9 12 14 15 19/30 Write block 85 len 32768 data: 15 12 10 11 13 14 12 12 20/30 Read block 428 len 4096 21/30 Read block 33 len 12288 22/30 Write block 239 len 36864 data: 13 14 13 12 13 15 16 13 14 23/30 Read block 39 len 28672 24/30 Read block 131 len 32768 25/30 Read block 296 len 20480 26/30 Read block 434 len 20480 27/30 Read block 151 len 36864 28/30 Read block 491 len 12288 29/30 Read block 374 len 12288 30/30 Read block 377 len 36864 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 9 Done Done Done false 1015 Acked Write 2 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 9 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 16 16 15 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:35:12.228053082Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1675} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 181 181 181 F F F 1 020-039 118 118 118 177 177 177 F F F 2 040-059 122 122 122 187 187 187 F F F 3 060-079 114 114 114 171 171 171 T T T 4 080-099 122 122 122 187 187 187 T T T 5 100-119 122 122 122 188 188 188 T T T 6 120-139 119 119 119 181 181 181 F F F 7 140-159 120 120 120 182 182 182 F F F 8 160-179 120 120 120 182 182 182 F F F 9 180-199 119 119 119 181 181 181 F F F 10 200-219 119 119 119 181 181 181 F F F 11 220-239 122 122 122 186 186 186 T T T 12 240-259 120 120 120 183 183 183 T T T 13 260-279 115 115 115 173 173 173 F F F 14 280-299 113 113 113 170 170 170 F F F 15 300-319 116 116 116 175 175 175 F F F 16 320-339 122 122 122 187 187 187 F F F 17 340-359 120 120 120 183 183 183 F F F 18 360-379 118 118 118 178 178 178 F F F 19 380-399 115 115 115 173 173 173 T T T 20 400-419 116 116 116 174 174 174 F F F 21 420-439 118 118 118 180 180 180 F F F 22 440-459 118 118 118 180 180 180 F F F 23 460-479 120 120 120 182 182 182 F F F 24 480-499 118 118 118 178 178 178 F F F 25 500-519 120 120 120 183 183 183 F F F 26 520-539 119 119 119 181 181 181 F F F 27 540-559 120 120 120 183 183 183 F F F 28 560-579 121 121 121 185 185 185 F F F 29 580-599 120 120 120 184 184 184 F F F Max gen: 122, Max flush: 188 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 181 181 181 F F F 1 020-039 118 118 118 177 177 177 F F F 2 040-059 122 122 122 187 187 187 F F F 3 060-079 123 123 123 189 189 189 F F F 4 080-099 123 123 123 189 189 189 F F F 5 100-119 123 123 123 189 189 189 F F F 6 120-139 119 119 119 181 181 181 F F F 7 140-159 120 120 120 182 182 182 F F F 8 160-179 120 120 120 182 182 182 F F F 9 180-199 119 119 119 181 181 181 F F F 10 200-219 119 119 119 181 181 181 F F F 11 220-239 123 123 123 189 189 189 F F F 12 240-259 123 123 123 189 189 189 F F F 13 260-279 115 115 115 173 173 173 F F F 14 280-299 113 113 113 170 170 170 F F F 15 300-319 116 116 116 175 175 175 F F F 16 320-339 122 122 122 187 187 187 F F F 17 340-359 120 120 120 183 183 183 F F F 18 360-379 118 118 118 178 178 178 F F F 19 380-399 123 123 123 189 189 189 F F F 20 400-419 116 116 116 174 174 174 F F F 21 420-439 118 118 118 180 180 180 F F F 22 440-459 118 118 118 180 180 180 F F F 23 460-479 120 120 120 182 182 182 F F F 24 480-499 118 118 118 178 178 178 F F F 25 500-519 120 120 120 183 183 183 F F F 26 520-539 119 119 119 181 181 181 F F F 27 540-559 120 120 120 183 183 183 F F F 28 560-579 121 121 121 185 185 185 F F F 29 580-599 120 120 120 184 184 184 F F F Max gen: 123, Max flush: 189 Begin loop 62 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.532871218Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.576841628Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.577436871Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.577472096Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.577487424Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.577502421Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683} {"msg":"Crucible 1f0e8c9e-e5d5-4084-b2a1-b3d820e7746e has session id: 84315a2d-75c9-4c7a-bee7-ec9892897224","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.577899873Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"Upstairs opts: Upstairs UUID: 1f0e8c9e-e5d5-4084-b2a1-b3d820e7746e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.577945857Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"Crucible stats registered with UUID: 1f0e8c9e-e5d5-4084-b2a1-b3d820e7746e","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.577964584Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.608565726Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.608608479Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683} {"msg":"1f0e8c9e-e5d5-4084-b2a1-b3d820e7746e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.608639495Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.60867005Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"client":"0","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.608717144Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"client":"0","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.608754948Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"client":"1","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.608868541Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"client":"1","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.608912355Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"client":"2","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.608941961Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"client":"2","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.608968107Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.608994153Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.60914819Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"ds_connection connected from Ok(127.0.0.1:51223)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.609181845Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"ds_connection connected from Ok(127.0.0.1:53687)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.609203812Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"ds_connection connected from Ok(127.0.0.1:52734)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.609230109Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.609747193Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"client":"2","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.609778738Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"client":"2","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.609809434Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"client":"0","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.60983966Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"client":"0","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.609867936Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"client":"1","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.609897201Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"client":"1","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.610360633Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"client":"2","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.610481606Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"client":"1","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.610504163Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"client":"0","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.610989952Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"client":"2","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.611031146Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"client":"2","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.611045983Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.611129661Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"client":"1","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.611151538Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"client":"1","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.611165446Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.611178134Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"client":"0","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.611192182Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"client":"0","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"[0]R flush_numbers[0..12]: [181, 177, 187, 189, 189, 189, 181, 182, 182, 181, 181, 189]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.611210409Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"":"downstairs","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"[0]R generation[0..12]: [119, 118, 122, 123, 123, 123, 119, 120, 120, 119, 119, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.611239785Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"":"downstairs","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.611255463Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"":"downstairs","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"[1]R flush_numbers[0..12]: [181, 177, 187, 189, 189, 189, 181, 182, 182, 181, 181, 189]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.611269671Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"":"downstairs","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"[1]R generation[0..12]: [119, 118, 122, 123, 123, 123, 119, 120, 120, 119, 119, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.611283649Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"":"downstairs","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.611297867Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"":"downstairs","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"[2]R flush_numbers[0..12]: [181, 177, 187, 189, 189, 189, 181, 182, 182, 181, 181, 189]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.611311865Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"":"downstairs","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"[2]R generation[0..12]: [119, 118, 122, 123, 123, 123, 119, 120, 120, 119, 119, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.611325723Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"":"downstairs","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.61134698Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"":"downstairs","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.611360627Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"":"downstairs","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.611374085Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"":"downstairs","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"Next flush: 190","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.611387374Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"":"downstairs","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.611400642Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"":"downstairs","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.61141422Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.611428477Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"client":"0","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.611442215Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"client":"1","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.611455464Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"client":"2","":"downstairs","session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.611468852Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.61148141Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"1f0e8c9e-e5d5-4084-b2a1-b3d820e7746e is now active with session: 84315a2d-75c9-4c7a-bee7-ec9892897224","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.611494348Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.611506546Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683,"session_id":"84315a2d-75c9-4c7a-bee7-ec9892897224"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:16.611519074Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 299 len 36864 data: 6 7 8 9 6 6 5 5 5 02/30 Read block 492 len 40960 03/30 Read block 464 len 36864 04/30 Read block 124 len 36864 05/30 Read block 461 len 32768 06/30 Read block 411 len 24576 07/30 Write block 99 len 12288 data: 19 16 15 08/30 Write block 129 len 12288 data: 13 13 13 09/30 Read block 24 len 16384 10/30 Write block 342 len 24576 data: 9 7 8 10 11 10 11/30 Read block 220 len 12288 12/30 Flush 13/30 Write block 213 len 24576 data: 9 10 8 8 8 7 14/30 Write block 304 len 24576 data: 7 6 6 6 6 6 15/30 Read block 80 len 36864 16/30 Write block 186 len 4096 data: 17 17/30 Write block 158 len 8192 data: 10 11 18/30 Write block 161 len 40960 data: 13 14 12 15 14 13 14 9 8 6 19/30 Flush 20/30 Write block 413 len 32768 data: 16 18 16 15 13 11 14 11 21/30 Read block 194 len 4096 22/30 Write block 466 len 4096 data: 9 23/30 Read block 457 len 12288 24/30 Read block 43 len 36864 25/30 Flush 26/30 Write block 37 len 40960 data: 7 7 10 10 9 12 10 10 11 13 27/30 Read block 508 len 28672 28/30 Read block 88 len 32768 29/30 Read block 205 len 36864 30/30 Read block 129 len 24576 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 10 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 6 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 5 4 5 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:35:16.632602676Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1683} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 181 181 181 F F F 1 020-039 118 118 118 177 177 177 T T T 2 040-059 122 122 122 187 187 187 T T T 3 060-079 123 123 123 189 189 189 F F F 4 080-099 124 124 124 190 190 190 F F F 5 100-119 124 124 124 190 190 190 F F F 6 120-139 124 124 124 190 190 190 F F F 7 140-159 124 124 124 191 191 191 F F F 8 160-179 124 124 124 191 191 191 F F F 9 180-199 124 124 124 191 191 191 F F F 10 200-219 124 124 124 191 191 191 F F F 11 220-239 123 123 123 189 189 189 F F F 12 240-259 123 123 123 189 189 189 F F F 13 260-279 115 115 115 173 173 173 F F F 14 280-299 124 124 124 190 190 190 F F F 15 300-319 124 124 124 191 191 191 F F F 16 320-339 122 122 122 187 187 187 F F F 17 340-359 124 124 124 190 190 190 F F F 18 360-379 118 118 118 178 178 178 F F F 19 380-399 123 123 123 189 189 189 F F F 20 400-419 124 124 124 192 192 192 F F F 21 420-439 124 124 124 192 192 192 F F F 22 440-459 118 118 118 180 180 180 F F F 23 460-479 124 124 124 192 192 192 F F F 24 480-499 118 118 118 178 178 178 F F F 25 500-519 120 120 120 183 183 183 F F F 26 520-539 119 119 119 181 181 181 F F F 27 540-559 120 120 120 183 183 183 F F F 28 560-579 121 121 121 185 185 185 F F F 29 580-599 120 120 120 184 184 184 F F F Max gen: 124, Max flush: 192 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 181 181 181 F F F 1 020-039 125 125 125 193 193 193 F F F 2 040-059 125 125 125 193 193 193 F F F 3 060-079 123 123 123 189 189 189 F F F 4 080-099 124 124 124 190 190 190 F F F 5 100-119 124 124 124 190 190 190 F F F 6 120-139 124 124 124 190 190 190 F F F 7 140-159 124 124 124 191 191 191 F F F 8 160-179 124 124 124 191 191 191 F F F 9 180-199 124 124 124 191 191 191 F F F 10 200-219 124 124 124 191 191 191 F F F 11 220-239 123 123 123 189 189 189 F F F 12 240-259 123 123 123 189 189 189 F F F 13 260-279 115 115 115 173 173 173 F F F 14 280-299 124 124 124 190 190 190 F F F 15 300-319 124 124 124 191 191 191 F F F 16 320-339 122 122 122 187 187 187 F F F 17 340-359 124 124 124 190 190 190 F F F 18 360-379 118 118 118 178 178 178 F F F 19 380-399 123 123 123 189 189 189 F F F 20 400-419 124 124 124 192 192 192 F F F 21 420-439 124 124 124 192 192 192 F F F 22 440-459 118 118 118 180 180 180 F F F 23 460-479 124 124 124 192 192 192 F F F 24 480-499 118 118 118 178 178 178 F F F 25 500-519 120 120 120 183 183 183 F F F 26 520-539 119 119 119 181 181 181 F F F 27 540-559 120 120 120 183 183 183 F F F 28 560-579 121 121 121 185 185 185 F F F 29 580-599 120 120 120 184 184 184 F F F Max gen: 125, Max flush: 193 Begin loop 63 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.885170569Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.930746454Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.931363144Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.931399639Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.931414557Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.931429354Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691} {"msg":"Crucible 8cad0acb-8acd-46eb-9983-95be1509f724 has session id: 52275b2a-d5f9-4daf-a02e-16accd30b1c7","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.931901696Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"Upstairs opts: Upstairs UUID: 8cad0acb-8acd-46eb-9983-95be1509f724, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.93194244Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"Crucible stats registered with UUID: 8cad0acb-8acd-46eb-9983-95be1509f724","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.931958987Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.963698013Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.963729608Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691} {"msg":"8cad0acb-8acd-46eb-9983-95be1509f724 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.963763893Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.963780481Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"client":"0","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.963809137Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"client":"0","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.963831773Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"client":"1","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.963849401Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"client":"1","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.963866278Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"client":"2","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.963883466Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"client":"2","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.963954285Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.96399414Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.964023295Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"ds_connection connected from Ok(127.0.0.1:39123)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.964052001Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"ds_connection connected from Ok(127.0.0.1:56779)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.964080477Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"ds_connection connected from Ok(127.0.0.1:61393)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.964108913Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.964835487Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"client":"2","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.964875321Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"client":"2","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.964908506Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"client":"0","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.964939812Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"client":"0","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.965036938Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"client":"1","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.96508542Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"client":"1","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.965466485Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"client":"0","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.96549608Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"client":"2","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.965512358Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"client":"1","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.966086324Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"client":"0","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.966127098Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"client":"0","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.966157964Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.96618405Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"client":"1","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.966208007Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"client":"1","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.966231663Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.9662538Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"client":"2","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.966279406Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"client":"2","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"[0]R flush_numbers[0..12]: [181, 193, 193, 189, 190, 190, 190, 191, 191, 191, 191, 189]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.966305392Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"":"downstairs","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"[0]R generation[0..12]: [119, 125, 125, 123, 124, 124, 124, 124, 124, 124, 124, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.966329149Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"":"downstairs","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.966354095Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"":"downstairs","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"[1]R flush_numbers[0..12]: [181, 193, 193, 189, 190, 190, 190, 191, 191, 191, 191, 189]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.966381831Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"":"downstairs","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"[1]R generation[0..12]: [119, 125, 125, 123, 124, 124, 124, 124, 124, 124, 124, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.966413117Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"":"downstairs","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.966442702Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"":"downstairs","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"[2]R flush_numbers[0..12]: [181, 193, 193, 189, 190, 190, 190, 191, 191, 191, 191, 189]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.966471978ZWait for a query_work_queue command to finish before sending IO ","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"":"downstairs","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"[2]R generation[0..12]: [119, 125, 125, 123, 124, 124, 124, 124, 124, 124, 124, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.966520101Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"":"downstairs","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.966550876Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"":"downstairs","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.966572453Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"":"downstairs","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.966586411Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"":"downstairs","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"Next flush: 194","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.966599479Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"":"downstairs","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.966612577Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"":"downstairs","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.966625795Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.966637994Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"client":"0","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.966651292Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"client":"1","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.96666454Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"client":"2","":"downstairs","session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.966678118Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.966693365Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"8cad0acb-8acd-46eb-9983-95be1509f724 is now active with session: 52275b2a-d5f9-4daf-a02e-16accd30b1c7","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.966706774Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.966719122Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691,"session_id":"52275b2a-d5f9-4daf-a02e-16accd30b1c7"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:20.9667317Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 492 len 40960 data: 11 12 11 12 12 11 11 13 13 13 03/30 Read block 269 len 16384 04/30 Read block 380 len 16384 05/30 Write block 302 len 4096 data: 10 06/30 Read block 229 len 28672 07/30 Read block 165 len 40960 08/30 Read block 508 len 16384 09/30 Write block 116 len 12288 data: 10 13 15 10/30 Write block 519 len 4096 data: 8 11/30 Write block 514 len 4096 data: 10 12/30 Write block 375 len 12288 data: 9 9 10 13/30 Write block 321 len 4096 data: 9 14/30 Write block 218 len 16384 data: 8 7 7 9 15/30 Read block 339 len 36864 16/30 Write block 515 len 40960 data: 6 8 8 9 9 10 10 10 10 11 17/30 Read block 309 len 24576 18/30 Write block 131 len 8192 data: 14 13 19/30 Read block 238 len 24576 20/30 Read block 48 len 16384 21/30 Write block 262 len 32768 data: 13 13 12 9 12 12 13 11 22/30 Write block 53 len 12288 data: 12 12 9 23/30 Flush 24/30 Write block 321 len 40960 data: 10 9 10 10 9 10 9 7 7 8 25/30 Write block 292 len 8192 data: 7 8 26/30 Write block 405 len 40960 data: 8 10 10 11 12 14 16 16 17 19 27/30 Write block 193 len 36864 data: 8 8 8 10 10 9 10 10 9 28/30 Read block 529 len 12288 29/30 Read block 339 len 32768 30/30 Write block 508 len 36864 data: 9 8 8 9 8 9 11 7 9 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 10 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:35:20.985145281Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1691} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 181 181 181 F F F 1 020-039 125 125 125 193 193 193 F F F 2 040-059 126 126 126 195 195 195 F F F 3 060-079 123 123 123 189 189 189 F F F 4 080-099 124 124 124 190 190 190 F F F 5 100-119 126 126 126 195 195 195 F F F 6 120-139 126 126 126 195 195 195 F F F 7 140-159 124 124 124 191 191 191 F F F 8 160-179 124 124 124 191 191 191 F F F 9 180-199 124 124 124 191 191 191 T T T 10 200-219 126 126 126 195 195 195 T T T 11 220-239 126 126 126 195 195 195 F F F 12 240-259 123 123 123 189 189 189 F F F 13 260-279 126 126 126 195 195 195 F F F 14 280-299 124 124 124 190 190 190 T T T 15 300-319 126 126 126 195 195 195 F F F 16 320-339 126 126 126 195 195 195 T T T 17 340-359 124 124 124 190 190 190 F F F 18 360-379 126 126 126 195 195 195 F F F 19 380-399 123 123 123 189 189 189 F F F 20 400-419 124 124 124 192 192 192 T T T 21 420-439 124 124 124 192 192 192 F F F 22 440-459 118 118 118 180 180 180 F F F 23 460-479 124 124 124 192 192 192 F F F 24 480-499 126 126 126 195 195 195 F F F 25 500-519 126 126 126 195 195 195 T T T 26 520-539 126 126 126 195 195 195 F F F 27 540-559 120 120 120 183 183 183 F F F 28 560-579 121 121 121 185 185 185 F F F 29 580-599 120 120 120 184 184 184 F F F Max gen: 126, Max flush: 195 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 181 181 181 F F F 1 020-039 125 125 125 193 193 193 F F F 2 040-059 126 126 126 195 195 195 F F F 3 060-079 123 123 123 189 189 189 F F F 4 080-099 124 124 124 190 190 190 F F F 5 100-119 126 126 126 195 195 195 F F F 6 120-139 126 126 126 195 195 195 F F F 7 140-159 124 124 124 191 191 191 F F F 8 160-179 124 124 124 191 191 191 F F F 9 180-199 127 127 127 196 196 196 F F F 10 200-219 127 127 127 196 196 196 F F F 11 220-239 126 126 126 195 195 195 F F F 12 240-259 123 123 123 189 189 189 F F F 13 260-279 126 126 126 195 195 195 F F F 14 280-299 127 127 127 196 196 196 F F F 15 300-319 126 126 126 195 195 195 F F F 16 320-339 127 127 127 196 196 196 F F F 17 340-359 124 124 124 190 190 190 F F F 18 360-379 126 126 126 195 195 195 F F F 19 380-399 123 123 123 189 189 189 F F F 20 400-419 127 127 127 196 196 196 F F F 21 420-439 124 124 124 192 192 192 F F F 22 440-459 118 118 118 180 180 180 F F F 23 460-479 124 124 124 192 192 192 F F F 24 480-499 126 126 126 195 195 195 F F F 25 500-519 127 127 127 196 196 196 F F F 26 520-539 126 126 126 195 195 195 F F F 27 540-559 120 120 120 183 183 183 F F F 28 560-579 121 121 121 185 185 185 F F F 29 580-599 120 120 120 184 184 184 F F F Max gen: 127, Max flush: 196 Begin loop 64 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.259460941Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:35:25.302088357Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.302462733Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.334407968Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.33501544Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.335052234Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.335066472Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.3350814Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699} {"msg":"Crucible c2f7bc4e-7aae-430c-a58c-94f47cbc11e3 has session id: d17e915d-48ca-48e2-864f-a3ae4b79045d","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.335503528Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"Upstairs opts: Upstairs UUID: c2f7bc4e-7aae-430c-a58c-94f47cbc11e3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.335550461Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"Crucible stats registered with UUID: c2f7bc4e-7aae-430c-a58c-94f47cbc11e3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.335576038Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.366406556Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.366437681Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699} {"msg":"c2f7bc4e-7aae-430c-a58c-94f47cbc11e3 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.366505731Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.366529248Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"client":"0","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.366555934Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"client":"0","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.366573612Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"client":"1","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.366590469Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"client":"1","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.366612286Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"client":"2","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.366725749Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"client":"2","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.366760954Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.36678933Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.366818396Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"ds_connection connected from Ok(127.0.0.1:48183)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.366849221Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"ds_connection connected from Ok(127.0.0.1:42966)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.366878417Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"ds_connection connected from Ok(127.0.0.1:56689)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.366907793Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.367577115Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"client":"1","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.367645895Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"client":"1","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.36768192Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"client":"2","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.367702537Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"client":"2","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.367724853Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"client":"0","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.36774677Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"client":"0","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.368241778Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"client":"1","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.368277213Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"client":"2","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.368424731Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"client":"0","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.368818984Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"client":"2","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.368854339Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"client":"2","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.368878335Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.368901652Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"client":"1","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.368928728Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"client":"1","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.368955994Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.36898218Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"client":"0","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.369006176Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"client":"0","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"[0]R flush_numbers[0..12]: [181, 193, 195, 189, 190, 195, 195, 191, 191, 196, 196, 195]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.369030853Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"":"downstairs","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"[0]R generation[0..12]: [119, 125, 126, 123, 124, 126, 126, 124, 124, 127, 127, 126]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.369056519Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"":"downstairs","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.369082925Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"":"downstairs","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"[1]R flush_numbers[0..12]: [181, 193, 195, 189, 190, 195, 195, 191, 191, 196, 196, 195]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.369107811Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"":"downstairs","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"[1]R generation[0..12]: [119, 125, 126, 123, 124, 126, 126, 124, 124, 127, 127, 126]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.369131768Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"":"downstairs","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.369161474Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"":"downstairs","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"[2]R flush_numbers[0..12]: [181, 193, 195, 189, 190, 195, 195, 191, 191, 196, 196, 195]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.369190549Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"":"downstairs","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"[2]R generation[0..12]: [119, 125, 126, 123, 124, 126, 126, 124, 124, 127, 127, 126]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.369216826Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"":"downstairs","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.369245701Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"":"downstairs","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.369281096Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"":"downstairs","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.369310242Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"":"downstairs","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"Next flush: 197","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.369338848Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"":"downstairs","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.369365934Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"":"downstairs","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.36939409Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.369419736Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"client":"0","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.369448452Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"clientDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":"1","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.369481947Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"client":"2","":"downstairs","session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.369509073Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.36952638Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"c2f7bc4e-7aae-430c-a58c-94f47cbc11e3 is now active with session: d17e915d-48ca-48e2-864f-a3ae4b79045d","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.369542708Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.369555386Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699,"session_id":"d17e915d-48ca-48e2-864f-a3ae4b79045d"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:25.369567754Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 205 len 8192 data: 5 6 02/30 Read block 380 len 28672 03/30 Read block 110 len 20480 04/30 Read block 88 len 40960 05/30 Write block 283 len 20480 data: 8 6 8 7 7 06/30 Write block 174 len 28672 data: 6 8 11 10 10 12 9 07/30 Read block 282 len 32768 08/30 Read block 235 len 40960 09/30 Write block 471 len 32768 data: 12 12 13 9 9 12 10 8 10/30 Flush 11/30 Write block 177 len 8192 data: 11 11 12/30 Read block 202 len 8192 13/30 Read block 348 len 24576 14/30 Read block 285 len 32768 15/30 Read block 276 len 20480 16/30 Write block 417 len 12288 data: 14 12 15 17/30 Write block 454 len 20480 data: 12 13 11 15 14 18/30 Write block 521 len 4096 data: 11 19/30 Write block 402 len 4096 data: 9 20/30 Read block 45 len 28672 21/30 Write block 140 len 32768 data: 7 9 10 11 11 13 14 15 22/30 Write block 544 len 24576 data: 10 10 12 12 15 12 23/30 Read block 19 len 4096 24/30 Read block 82 len 4096 25/30 Write block 503 len 8192 data: 10 10 26/30 Write block 71 len 28672 data: 8 10 10 12 13 14 13 27/30 Read block 21 len 32768 28/30 Read block 493 len 16384 29/30 Read block 126 len 36864 30/30 Write block 70 len 32768 data: 11 9 11 11 13 14 15 14 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 2 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 19 19 19 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1009 1009 1009 Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:35:25.388397565Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1699} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 181 181 181 F F F 1 020-039 125 125 125 193 193 193 F F F 2 040-059 126 126 126 195 195 195 F F F 3 060-079 123 123 123 189 189 189 T T T 4 080-099 124 124 124 190 190 190 F F F 5 100-119 126 126 126 195 195 195 F F F 6 120-139 126 126 126 195 195 195 F F F 7 140-159 124 124 124 191 191 191 T T T 8 160-179 128 128 128 197 197 197 T T T 9 180-199 128 128 128 197 197 197 F F F 10 200-219 128 128 128 197 197 197 F F F 11 220-239 126 126 126 195 195 195 F F F 12 240-259 123 123 123 189 189 189 F F F 13 260-279 126 126 126 195 195 195 F F F 14 280-299 128 128 128 197 197 197 F F F 15 300-319 126 126 126 195 195 195 F F F 16 320-339 127 127 127 196 196 196 F F F 17 340-359 124 124 124 190 190 190 F F F 18 360-379 126 126 126 195 195 195 F F F 19 380-399 123 123 123 189 189 189 F F F 20 400-419 127 127 127 196 196 196 T T T 21 420-439 124 124 124 192 192 192 F F F 22 440-459 118 118 118 180 180 180 T T T 23 460-479 128 128 128 197 197 197 F F F 24 480-499 126 126 126 195 195 195 F F F 25 500-519 127 127 127 196 196 196 T T T 26 520-539 126 126 126 195 195 195 T T T 27 540-559 120 120 120 183 183 183 T T T 28 560-579 121 121 121 185 185 185 F F F 29 580-599 120 120 120 184 184 184 F F F Max gen: 128, Max flush: 197 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 181 181 181 F F F 1 020-039 125 125 125 193 193 193 F F F 2 040-059 126 126 126 195 195 195 F F F 3 060-079 129 129 129 198 198 198 F F F 4 080-099 124 124 124 190 190 190 F F F 5 100-119 126 126 126 195 195 195 F F F 6 120-139 126 126 126 195 195 195 F F F 7 140-159 129 129 129 198 198 198 F F F 8 160-179 129 129 129 198 198 198 F F F 9 180-199 128 128 128 197 197 197 F F F 10 200-219 128 128 128 197 197 197 F F F 11 220-239 126 126 126 195 195 195 F F F 12 240-259 123 123 123 189 189 189 F F F 13 260-279 126 126 126 195 195 195 F F F 14 280-299 128 128 128 197 197 197 F F F 15 300-319 126 126 126 195 195 195 F F F 16 320-339 127 127 127 196 196 196 F F F 17 340-359 124 124 124 190 190 190 F F F 18 360-379 126 126 126 195 195 195 F F F 19 380-399 123 123 123 189 189 189 F F F 20 400-419 129 129 129 198 198 198 F F F 21 420-439 124 124 124 192 192 192 F F F 22 440-459 129 129 129 198 198 198 F F F 23 460-479 128 128 128 197 197 197 F F F 24 480-499 126 126 126 195 195 195 F F F 25 500-519 129 129 129 198 198 198 F F F 26 520-539 129 129 129 198 198 198 F F F 27 540-559 129 129 129 198 198 198 F F F 28 560-579 121 121 121 185 185 185 F F F 29 580-599 120 120 120 184 184 184 F F F Max gen: 129, Max flush: 198 Begin loop 65 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.703199514Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.746787459Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.747390111Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.747426776Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.747441274Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.747456091Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707} {"msg":"Crucible 5b079c9e-7e65-4b55-854b-526f15898af6 has session id: f2446673-d01b-4e67-810d-ae82467902e1","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.74788014Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"Upstairs opts: Upstairs UUID: 5b079c9e-7e65-4b55-854b-526f15898af6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.747929233Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"Crucible stats registered with UUID: 5b079c9e-7e65-4b55-854b-526f15898af6","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.747954859Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.779185458Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.779219013Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707} {"msg":"5b079c9e-7e65-4b55-854b-526f15898af6 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.779235561Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.779252269Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"client":"0","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.779280264Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"client":"0","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.779298352Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"client":"1","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.779315529Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"client":"1","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.779336286Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"client":"2","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.779354184Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"client":"2","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.779371021Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.779388309Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.779405576Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:38265)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.77958474Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:42862)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.779732029Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:42215)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.779769193Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.780356018Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"client":"0","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.780385613Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"client":"0","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.780401121Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"client":"1","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.780415139Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"client":"1","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.780430387Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"client":"2","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.780500386Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"client":"2","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.78108925Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"client":"1","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.781120496Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"client":"0","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.781141653Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"client":"2","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.781686473Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"client":"1","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.781715609Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"client":"1","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.781731217Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.781744465Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"client":"2","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.781758093Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"client":"2","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.781771461Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.781783789Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"client":"0","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.781797107Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"client":"0","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"[0]R flush_numbers[0..12]: [181, 193, 195, 198, 190, 195, 195, 198, 198, 197, 197, 195]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.781810995Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"":"downstairs","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"[0]R generation[0..12]: [119, 125, 126, 129, 124, 126, 126, 129, 129, 128, 128, 126]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.781825163Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"":"downstairs","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.781839281Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"":"downstairs","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"[1]R flush_numbers[0..12]: [181, 193, 195, 198, 190, 195, 195, 198, 198, 197, 197, 195]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.781853229Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"":"downstairs","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"[1]R generation[0..12]: [119, 125, 126, 129, 124, 126, 126, 129, 129, 128, 128, 126]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.781867067Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"":"downstairs","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.781886564Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"":"downstairs","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"[2]R flush_numbers[0..12]: [181, 193, 195, 198, 190, 195, 195, 198, 198, 197, 197, 195]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.781901082Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"":"downstairs","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"[2]R generation[0..12]: [119, 125, 126, 129, 124, 126, 126, 129, 129, 128, 128, 126]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.78191504Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"":"downstairs","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","Wait for a query_work_queue command to finish before sending IO v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.781937327Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"":"downstairs","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.781950775Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"":"downstairs","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.781964083Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"":"downstairs","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"Next flush: 199","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.781977041Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"":"downstairs","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.781990489Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"":"downstairs","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.782015435Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.782031293Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"client":"0","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.782045151Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"client":"1","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.782058629Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"client":"2","":"downstairs","session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.782072017Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.782084605Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"5b079c9e-7e65-4b55-854b-526f15898af6 is now active with session: f2446673-d01b-4e67-810d-ae82467902e1","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.782097623Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.782112261Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707,"session_id":"f2446673-d01b-4e67-810d-ae82467902e1"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:29.782125009Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 256 len 12288 data: 14 14 14 02/30 Read block 31 len 8192 03/30 Flush 04/30 Write block 585 len 36864 data: 10 11 13 12 14 13 12 12 11 05/30 Read block 524 len 32768 06/30 Read block 212 len 16384 07/30 Write block 83 len 28672 data: 9 8 16 13 11 12 14 08/30 Read block 193 len 40960 09/30 Write block 246 len 24576 data: 14 15 13 11 13 13 10/30 Write block 417 len 24576 data: 15 13 16 12 13 10 11/30 Read block 256 len 28672 12/30 Read block 388 len 20480 13/30 Read block 109 len 32768 14/30 Read block 151 len 16384 15/30 Write block 586 len 20480 data: 12 14 13 15 14 16/30 Write block 318 len 4096 data: 14 17/30 Write block 174 len 16384 data: 7 9 12 12 18/30 Write block 514 len 12288 data: 12 8 10 19/30 Read block 49 len 24576 20/30 Read block 178 len 28672 21/30 Read block 558 len 36864 22/30 Read block 423 len 4096 23/30 Write block 112 len 36864 data: 7 8 9 9 11 14 16 16 13 24/30 Read block 242 len 4096 25/30 Flush 26/30 Write block 340 len 36864 data: 7 9 10 8 9 11 12 11 10 27/30 Write block 365 len 28672 data: 11 10 10 10 9 12 12 28/30 Write block 397 len 12288 data: 11 12 12 29/30 Read block 36 len 8192 30/30 Write block 317 len 12288 data: 13 15 12 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 9 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:35:29.80101847Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1707} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 181 181 181 F F F 1 020-039 125 125 125 193 193 193 F F F 2 040-059 126 126 126 195 195 195 F F F 3 060-079 129 129 129 198 198 198 F F F 4 080-099 130 130 130 200 200 200 F F F 5 100-119 130 130 130 200 200 200 F F F 6 120-139 130 130 130 200 200 200 F F F 7 140-159 129 129 129 198 198 198 F F F 8 160-179 130 130 130 200 200 200 F F F 9 180-199 128 128 128 197 197 197 F F F 10 200-219 128 128 128 197 197 197 F F F 11 220-239 126 126 126 195 195 195 F F F 12 240-259 130 130 130 200 200 200 F F F 13 260-279 126 126 126 195 195 195 F F F 14 280-299 128 128 128 197 197 197 F F F 15 300-319 130 130 130 200 200 200 T T T 16 320-339 127 127 127 196 196 196 F F F 17 340-359 124 124 124 190 190 190 T T T 18 360-379 126 126 126 195 195 195 T T T 19 380-399 123 123 123 189 189 189 T T T 20 400-419 130 130 130 200 200 200 F F F 21 420-439 130 130 130 200 200 200 F F F 22 440-459 129 129 129 198 198 198 F F F 23 460-479 128 128 128 197 197 197 F F F 24 480-499 126 126 126 195 195 195 F F F 25 500-519 130 130 130 200 200 200 F F F 26 520-539 129 129 129 198 198 198 F F F 27 540-559 129 129 129 198 198 198 F F F 28 560-579 121 121 121 185 185 185 F F F 29 580-599 130 130 130 200 200 200 F F F Max gen: 130, Max flush: 200 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 181 181 181 F F F 1 020-039 125 125 125 193 193 193 F F F 2 040-059 126 126 126 195 195 195 F F F 3 060-079 129 129 129 198 198 198 F F F 4 080-099 130 130 130 200 200 200 F F F 5 100-119 130 130 130 200 200 200 F F F 6 120-139 130 130 130 200 200 200 F F F 7 140-159 129 129 129 198 198 198 F F F 8 160-179 130 130 130 200 200 200 F F F 9 180-199 128 128 128 197 197 197 F F F 10 200-219 128 128 128 197 197 197 F F F 11 220-239 126 126 126 195 195 195 F F F 12 240-259 130 130 130 200 200 200 F F F 13 260-279 126 126 126 195 195 195 F F F 14 280-299 128 128 128 197 197 197 F F F 15 300-319 131 131 131 201 201 201 F F F 16 320-339 127 127 127 196 196 196 F F F 17 340-359 131 131 131 201 201 201 F F F 18 360-379 131 131 131 201 201 201 F F F 19 380-399 131 131 131 201 201 201 F F F 20 400-419 130 130 130 200 200 200 F F F 21 420-439 130 130 130 200 200 200 F F F 22 440-459 129 129 129 198 198 198 F F F 23 460-479 128 128 128 197 197 197 F F F 24 480-499 126 126 126 195 195 195 F F F 25 500-519 130 130 130 200 200 200 F F F 26 520-539 129 129 129 198 198 198 F F F 27 540-559 129 129 129 198 198 198 F F F 28 560-579 121 121 121 185 185 185 F F F 29 580-599 130 130 130 200 200 200 F F F Max gen: 131, Max flush: 201 Begin loop 66 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.061265414Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.104688374Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.105326771Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.105363306Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.105378444Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.105397101Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715} {"msg":"Crucible 1bbd9356-744c-44d7-b377-d691d77e8278 has session id: 8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.105863543Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"Upstairs opts: Upstairs UUID: 1bbd9356-744c-44d7-b377-d691d77e8278, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.105900007Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"Crucible stats registered with UUID: 1bbd9356-744c-44d7-b377-d691d77e8278","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.105916305Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.137193718Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.137226303Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715} {"msg":"1bbd9356-744c-44d7-b377-d691d77e8278 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.137257559Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.137274576Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"client":"0","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.137301522Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"client":"0","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.1373195Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"client":"1","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.137336737Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"client":"1","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.137365523Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"client":"2","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.137401088Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"client":"2","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.137430013Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.137459289Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.137499213Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"ds_connection connected from Ok(127.0.0.1:58003)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.137534448Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"ds_connection connected from Ok(127.0.0.1:43892)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.137581042Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"ds_connection connected from Ok(127.0.0.1:62828)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.137619516Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.13834707Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"client":"1","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.138375915Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"client":"1","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.138392193Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"client":"0","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.138406431Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"client":"0","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.138466942Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"client":"2","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.138488219Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"client":"2","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.139052736Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"client":"2","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.139198985Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"client":"1","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.139240129Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"client":"0","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.139617214Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"client":"2","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.13964501Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"client":"2","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.139662998Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.139738766Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"client":"1","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.139759073Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"client":"1","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.139772991Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.13978533Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"client":"0","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.139798738Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"client":"0","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"[0]R flush_numbers[0..12]: [181, 193, 195, 198, 200, 200, 200, 198, 200, 197, 197, 195]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.139812865Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"":"downstairs","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"[0]R generation[0..12]: [119, 125, 126, 129, 130, 130, 130, 129, 130, 128, 128, 126]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.139826874Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"":"downstairs","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.139841112Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"":"downstairs","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"[1]R flush_numbers[0..12]: [181, 193, 195, 198, 200, 200, 200, 198, 200, 197, 197, 195]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.139855129Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"":"downstairs","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"[1]R generation[0..12]: [119, 125, 126, 129, 130, 130, 130, 129, 130, 128, 128, 126]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.139868907Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"":"downstairs","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.139882885Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"":"downstairs","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"[2]R flush_numbers[0..12]: [181, 193, 195, 198, 200, 200, 200, 198, 200, 197, 197, 195]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.139896993Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"":"downstairs","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"[2]R generation[0..12]: [119, 125, 126, 129, 130, 130, 130, 129, 130, 128, 128, 126]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.139910871Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"":"downstairs","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.139927879Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"":"downstairs","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.139941367Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"":"downstairs","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.139954655Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"":"downstairs","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"Next flush: 202","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:35:34.139967803Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"":"downstairs","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.1399873Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"":"downstairs","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.140000948Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.140013436Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"client":"0","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.140026944Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"client":"1","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.140040522Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"client":"2","":"downstairs","session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.14005412Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.140066519Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"1bbd9356-744c-44d7-b377-d691d77e8278 is now active with session: 8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.140079357Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.140091485Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715,"session_id":"8bc56c62-3fa4-4d5b-a1d2-9bd822d3f707"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:34.140103893Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 47 len 4096 data: 10 02/30 Read block 307 len 16384 03/30 Read block 349 len 16384 04/30 Flush 05/30 Write block 248 len 36864 data: 14 12 14 14 15 14 14 11 15 06/30 Write block 216 len 28672 data: 9 9 9 8 8 10 9 07/30 Write block 377 len 32768 data: 11 9 8 7 6 7 6 7 08/30 Read block 338 len 20480 09/30 Write block 306 len 28672 data: 7 7 7 7 8 9 10 10/30 Read block 279 len 24576 11/30 Read block 537 len 4096 12/30 Read block 246 len 20480 13/30 Read block 210 len 24576 14/30 Write block 8 len 8192 data: 6 10 15/30 Read block 412 len 28672 16/30 Read block 123 len 4096 17/30 Read block 370 len 40960 18/30 Write block 550 len 8192 data: 14 15 19/30 Read block 512 len 16384 20/30 Read block 230 len 40960 21/30 Write block 53 len 8192 data: 13 13 22/30 Read block 209 len 40960 23/30 Read block 140 len 32768 24/30 Write block 359 len 36864 data: 8 8 9 7 8 9 12 11 11 25/30 Write block 367 len 32768 data: 12 11 10 13 13 12 10 9 26/30 Write block 498 len 24576 data: 12 14 14 14 10 11 27/30 Read block 493 len 8192 28/30 Write block 364 len 12288 data: 10 13 12 29/30 Write block 448 len 4096 data: 12 30/30 Read block 9 len 4096 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 9 Done Done Done false 1005 Acked Write 7 Done Done Done false 1006 Acked Write 8 Done Done Done false 1007 Acked Read 5 Done Done Done false 1008 Acked Write 7 Done Done Done false 1009 Acked Read 6 Done Done Done false 1010 Acked Read 1 Done Done Done false 1011 Acked Read 5 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Write 2 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 3 Done Done Sent false 1028 Acked Write 1 Done Done Sent false 1029 Acked Read 1 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 3 3 Done 26 26 23 75 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:35:34.158779186Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1715} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 181 181 181 T T T 1 020-039 125 125 125 193 193 193 F F F 2 040-059 132 132 132 202 202 202 T T T 3 060-079 129 129 129 198 198 198 F F F 4 080-099 130 130 130 200 200 200 F F F 5 100-119 130 130 130 200 200 200 F F F 6 120-139 130 130 130 200 200 200 F F F 7 140-159 129 129 129 198 198 198 F F F 8 160-179 130 130 130 200 200 200 F F F 9 180-199 128 128 128 197 197 197 F F F 10 200-219 128 128 128 197 197 197 T T T 11 220-239 126 126 126 195 195 195 T T T 12 240-259 130 130 130 200 200 200 T T T 13 260-279 126 126 126 195 195 195 F F F 14 280-299 128 128 128 197 197 197 F F F 15 300-319 131 131 131 201 201 201 T T T 16 320-339 127 127 127 196 196 196 F F F 17 340-359 131 131 131 201 201 201 T T T 18 360-379 131 131 131 201 201 201 T T T 19 380-399 131 131 131 201 201 201 T T T 20 400-419 130 130 130 200 200 200 F F F 21 420-439 130 130 130 200 200 200 F F F 22 440-459 129 129 129 198 198 198 T T T 23 460-479 128 128 128 197 197 197 F F F 24 480-499 126 126 126 195 195 195 T T T 25 500-519 130 130 130 200 200 200 T T T 26 520-539 129 129 129 198 198 198 F F F 27 540-559 129 129 129 198 198 198 T T T 28 560-579 121 121 121 185 185 185 F F F 29 580-599 130 130 130 200 200 200 F F F Max gen: 132, Max flush: 202 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 203 203 203 F F F 1 020-039 125 125 125 193 193 193 F F F 2 040-059 133 133 133 203 203 203 F F F 3 060-079 129 129 129 198 198 198 F F F 4 080-099 130 130 130 200 200 200 F F F 5 100-119 130 130 130 200 200 200 F F F 6 120-139 130 130 130 200 200 200 F F F 7 140-159 129 129 129 198 198 198 F F F 8 160-179 130 130 130 200 200 200 F F F 9 180-199 128 128 128 197 197 197 F F F 10 200-219 133 133 133 203 203 203 F F F 11 220-239 133 133 133 203 203 203 F F F 12 240-259 133 133 133 203 203 203 F F F 13 260-279 126 126 126 195 195 195 F F F 14 280-299 128 128 128 197 197 197 F F F 15 300-319 133 133 133 203 203 203 F F F 16 320-339 127 127 127 196 196 196 F F F 17 340-359 133 133 133 203 203 203 F F F 18 360-379 133 133 133 203 203 203 F F F 19 380-399 133 133 133 203 203 203 F F F 20 400-419 130 130 130 200 200 200 F F F 21 420-439 130 130 130 200 200 200 F F F 22 440-459 133 133 133 203 203 203 F F F 23 460-479 128 128 128 197 197 197 F F F 24 480-499 133 133 133 203 203 203 F F F 25 500-519 133 133 133 203 203 203 F F F 26 520-539 129 129 129 198 198 198 F F F 27 540-559 133 133 133 203 203 203 F F F 28 560-579 121 121 121 185 185 185 F F F 29 580-599 130 130 130 200 200 200 F F F Max gen: 133, Max flush: 203 Begin loop 67 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.468856215Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.511915478Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.516086319Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.516132372Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.516155839Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.516174566Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723} {"msg":"Crucible 45133435-f217-4473-8752-d5eb3d07d622 has session id: d7a5bdbb-2439-4d13-8359-482746af25a2","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.516635429Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"Upstairs opts: Upstairs UUID: 45133435-f217-4473-8752-d5eb3d07d622, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.516672383Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"Crucible stats registered with UUID: 45133435-f217-4473-8752-d5eb3d07d622","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.516690251Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.54689585Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.546928096Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723} {"msg":"45133435-f217-4473-8752-d5eb3d07d622 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.54696327Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.546982028Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"client":"0","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.547010453Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"client":"0","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.54703085Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"client":"1","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.547050278Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"client":"1","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.547069725Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"client":"2","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.547088062Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"client":"2","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.54710547Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.547139325Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.5471746Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:61569)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.547208975Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:49815)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.547247289Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:55833)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.547284144Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.548084187Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"client":"1","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.548122551Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"client":"1","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.548153267Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"client":"0","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.548183542Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"client":"0","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.548215477Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"client":"2","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.548244303Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"client":"2","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.548718294Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"client":"1","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.548759058Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"client":"0","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.54888114Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"client":"2","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.549353031Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"client":"1","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.549382547Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"client":"1","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.549397065Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.549409973Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"client":"2","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.549423931Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"client":"2","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.549438769Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.549451237Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"client":"0","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.549464685Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"client":"0","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"[0]R flush_numbers[0..12]: [203, 193, 203, 198, 200, 200, 200, 198, 200, 197, 203, 203]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.549478673Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"":"downstairs","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"[0]R generation[0..12]: [133, 125, 133, 129, 130, 130, 130, 129, 130, 128, 133, 133]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.549492781Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"":"downstairs","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.549507079Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"":"downstairs","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"[1]R flush_numbers[0..12]: [203, 193, 203, 198, 200, 200, 200, 198, 200, 197, 203, 203]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.549520937Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"":"downstairs","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"[1]R generation[0..12]: [133, 125, 133, 129, 130, 130, 130, 129, 130, 128, 133, 133]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.549534915Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"":"downstairs","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.549548963Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"":"downstairs","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"[2]R flush_numbers[0..12]: [203, 193, 203, 198, 200, 200, 200, 198, 200, 197, 203, 203]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.549562691Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"":"downstairs","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"[2]R generation[0..12]: [133, 125, 133, 129, 130, 130, 130, 129, 130, 128, 133, 133]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.549576559Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"":"downstairs","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.549592596Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"":"downstairs",""Wait for a query_work_queue command to finish before sending IO :"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.549614803Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"":"downstairs","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.549628431Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"":"downstairs","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"Next flush: 204","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.549641499Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"":"downstairs","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.549654687Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"":"downstairs","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.549668105Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.549680603Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"client":"0","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.549694122Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"client":"1","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.54970727Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"client":"2","":"downstairs","session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.549720918Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.549733456Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"45133435-f217-4473-8752-d5eb3d07d622 is now active with session: d7a5bdbb-2439-4d13-8359-482746af25a2","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.549746324Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.549758462Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723,"session_id":"d7a5bdbb-2439-4d13-8359-482746af25a2"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:38.54977093Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 55 len 16384 data: 10 10 12 13 02/30 Read block 2 len 20480 03/30 Write block 46 len 12288 data: 14 11 10 04/30 Write block 184 len 12288 data: 16 15 18 05/30 Read block 282 len 32768 06/30 Read block 401 len 8192 07/30 Read block 392 len 12288 08/30 Read block 253 len 24576 09/30 Write block 262 len 32768 data: 14 14 13 10 13 13 14 12 10/30 Write block 476 len 24576 data: 13 11 9 8 10 10 11/30 Read block 218 len 16384 12/30 Read block 264 len 28672 13/30 Flush 14/30 Write block 318 len 12288 data: 16 13 11 15/30 Write block 89 len 12288 data: 15 15 13 16/30 Read block 131 len 8192 17/30 Read block 546 len 16384 18/30 Write block 122 len 40960 data: 12 15 12 11 10 11 10 14 14 15 19/30 Write block 350 len 16384 data: 12 12 11 10 20/30 Read block 9 len 20480 21/30 Write block 589 len 8192 data: 16 15 22/30 Flush 23/30 Write block 304 len 20480 data: 8 7 8 8 8 24/30 Flush 25/30 Write block 68 len 28672 data: 14 11 12 10 12 12 14 26/30 Write block 359 len 16384 data: 9 9 10 8 27/30 Read block 554 len 12288 28/30 Read block 119 len 16384 29/30 Read block 326 len 24576 30/30 Read block 177 len 24576 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 7 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 6 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 5 6 6 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:35:38.56894934Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1723} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 203 203 203 F F F 1 020-039 125 125 125 193 193 193 F F F 2 040-059 134 134 134 204 204 204 F F F 3 060-079 129 129 129 198 198 198 T T T 4 080-099 134 134 134 205 205 205 F F F 5 100-119 130 130 130 200 200 200 F F F 6 120-139 134 134 134 205 205 205 F F F 7 140-159 129 129 129 198 198 198 F F F 8 160-179 130 130 130 200 200 200 F F F 9 180-199 134 134 134 204 204 204 F F F 10 200-219 133 133 133 203 203 203 F F F 11 220-239 133 133 133 203 203 203 F F F 12 240-259 133 133 133 203 203 203 F F F 13 260-279 134 134 134 204 204 204 F F F 14 280-299 128 128 128 197 197 197 F F F 15 300-319 134 134 134 206 206 206 F F F 16 320-339 134 134 134 205 205 205 F F F 17 340-359 134 134 134 205 205 205 T T T 18 360-379 133 133 133 203 203 203 T T T 19 380-399 133 133 133 203 203 203 F F F 20 400-419 130 130 130 200 200 200 F F F 21 420-439 130 130 130 200 200 200 F F F 22 440-459 133 133 133 203 203 203 F F F 23 460-479 134 134 134 204 204 204 F F F 24 480-499 134 134 134 204 204 204 F F F 25 500-519 133 133 133 203 203 203 F F F 26 520-539 129 129 129 198 198 198 F F F 27 540-559 133 133 133 203 203 203 F F F 28 560-579 121 121 121 185 185 185 F F F 29 580-599 134 134 134 205 205 205 F F F Max gen: 134, Max flush: 206 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 203 203 203 F F F 1 020-039 125 125 125 193 193 193 F F F 2 040-059 134 134 134 204 204 204 F F F 3 060-079 135 135 135 207 207 207 F F F 4 080-099 134 134 134 205 205 205 F F F 5 100-119 130 130 130 200 200 200 F F F 6 120-139 134 134 134 205 205 205 F F F 7 140-159 129 129 129 198 198 198 F F F 8 160-179 130 130 130 200 200 200 F F F 9 180-199 134 134 134 204 204 204 F F F 10 200-219 133 133 133 203 203 203 F F F 11 220-239 133 133 133 203 203 203 F F F 12 240-259 133 133 133 203 203 203 F F F 13 260-279 134 134 134 204 204 204 F F F 14 280-299 128 128 128 197 197 197 F F F 15 300-319 134 134 134 206 206 206 F F F 16 320-339 134 134 134 205 205 205 F F F 17 340-359 135 135 135 207 207 207 F F F 18 360-379 135 135 135 207 207 207 F F F 19 380-399 133 133 133 203 203 203 F F F 20 400-419 130 130 130 200 200 200 F F F 21 420-439 130 130 130 200 200 200 F F F 22 440-459 133 133 133 203 203 203 F F F 23 460-479 134 134 134 204 204 204 F F F 24 480-499 134 134 134 204 204 204 F F F 25 500-519 133 133 133 203 203 203 F F F 26 520-539 129 129 129 198 198 198 F F F 27 540-559 133 133 133 203 203 203 F F F 28 560-579 121 121 121 185 185 185 F F F 29 580-599 134 134 134 205 205 205 F F F Max gen: 135, Max flush: 207 Begin loop 68 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.825013705Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.868019325Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.868599011Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.868634406Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.868648993Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.868663641Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731} {"msg":"Crucible fa728112-bb57-4c41-82ea-609842d9b521 has session id: 7c78478b-0f17-4320-9cce-c259ca78df86","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.869056914Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"Upstairs opts: Upstairs UUID: fa728112-bb57-4c41-82ea-609842d9b521, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.869100317Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"Crucible stats registered with UUID: fa728112-bb57-4c41-82ea-609842d9b521","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.869121524Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.899846468Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.899879413Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731} {"msg":"fa728112-bb57-4c41-82ea-609842d9b521 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.899909989Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.899928086Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"client":"0","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.899956412Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"client":"0","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.899975449Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"client":"1","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.899993237Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"client":"1","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.900010124Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"client":"2","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.900028042Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"client":"2","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.900059817Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"":"io task","client":"2","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.900095912Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"":"io task","client":"0","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.900127657Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"":"io task","client":"1","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"ds_connection connected from Ok(127.0.0.1:48878)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.900166281Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"":"io task","client":"2","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"ds_connection connected from Ok(127.0.0.1:34453)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.900202256Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"":"io task","client":"1","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"ds_connection connected from Ok(127.0.0.1:37029)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.90024319Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"":"io task","client":"0","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.901003419Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"client":"2","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.901040424Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"client":"2","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.901072069Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"client":"0","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.901099565Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"client":"0","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.901218948Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"client":"1","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.90126883Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"client":"1","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.901858724Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"client":"2","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.901898149Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"client":"0","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.901928814Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"client":"1","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.902448178Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"client":"1","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.902477124Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"client":"1","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.902492652Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.90250579Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"client":"2","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.902520168Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"client":"2","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.902538135Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.902551163Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"client":"0","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.902564531Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"client":"0","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"[0]R flush_numbers[0..12]: [203, 193, 204, 207, 205, 200, 205, 198, 200, 204, 203, 203]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.902578459Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"":"downstairs","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"[0]R generation[0..12]: [133, 125, 134, 135, 134, 130, 134, 129, 130, 134, 133, 133]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.902592547Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"":"downstairs","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.902606715Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"":"downstairs","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"[1]R flush_numbers[0..12]: [203, 193, 204, 207, 205, 200, 205, 198, 200, 204, 203, 203]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.902620573Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"":"downstairs","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"[1]R generation[0..12]: [133, 125, 134, 135, 134, 130, 134, 129, 130, 134, 133, 133]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.902634411Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"":"downstairs","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.902648539Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"":"downstairs","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"[2]R flush_numbers[0..12]: [203, 193, 204, 207, 205, 200, 205, 198, 200, 204, 203, 203]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:35:42.902662387Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"":"downstairs","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"[2]R generation[0..12]: [133, 125, 134, 135, 134, 130, 134, 129, 130, 134, 133, 133]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.902681024Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"":"downstairs","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.902694972Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"":"downstairs","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.90270811Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"":"downstairs","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.902721488Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"":"downstairs","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"Next flush: 208","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.902736596Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"":"downstairs","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.902750194Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"":"downstairs","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.902763492Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.90277564Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"client":"0","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.902788819Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"client":"1","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.902802366Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"client":"2","":"downstairs","session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.902829022Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.902842491Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"fa728112-bb57-4c41-82ea-609842d9b521 is now active with session: 7c78478b-0f17-4320-9cce-c259ca78df86","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.902855479Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.902867787Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731,"session_id":"7c78478b-0f17-4320-9cce-c259ca78df86"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:42.902880085Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 336 len 16384 data: 6 6 7 8 02/30 Read block 414 len 4096 03/30 Write block 116 len 36864 data: 12 15 17 17 14 12 13 16 13 04/30 Read block 97 len 28672 05/30 Read block 511 len 8192 06/30 Read block 113 len 8192 07/30 Read block 179 len 24576 08/30 Read block 268 len 32768 09/30 Read block 378 len 40960 10/30 Read block 277 len 36864 11/30 Write block 371 len 32768 data: 14 13 11 10 10 10 12 10 12/30 Read block 67 len 24576 13/30 Write block 331 len 20480 data: 6 5 6 7 6 14/30 Write block 421 len 16384 data: 14 11 9 10 15/30 Write block 523 len 20480 data: 11 12 14 13 13 16/30 Write block 376 len 20480 data: 11 13 11 9 8 17/30 Flush 18/30 Write block 227 len 16384 data: 11 14 13 14 19/30 Write block 112 len 40960 data: 8 9 10 10 13 16 18 18 15 13 20/30 Write block 564 len 32768 data: 10 9 10 10 11 11 10 10 21/30 Write block 219 len 40960 data: 9 9 11 10 10 11 8 10 12 15 22/30 Read block 13 len 20480 23/30 Read block 436 len 12288 24/30 Flush 25/30 Write block 253 len 36864 data: 15 15 12 16 15 15 14 13 13 26/30 Read block 100 len 8192 27/30 Read block 473 len 32768 28/30 Read block 402 len 40960 29/30 Write block 577 len 12288 data: 7 10 11 30/30 Write block 275 len 36864 data: 9 8 8 7 5 6 5 6 9 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 9 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 3 Sent Done Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 1 2 5 Done 4 5 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:35:42.923626056Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1731} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 203 203 203 F F F 1 020-039 125 125 125 193 193 193 F F F 2 040-059 134 134 134 204 204 204 F F F 3 060-079 135 135 135 207 207 207 F F F 4 080-099 134 134 134 205 205 205 F F F 5 100-119 136 136 136 209 209 209 F F F 6 120-139 136 136 136 209 209 209 F F F 7 140-159 129 129 129 198 198 198 F F F 8 160-179 130 130 130 200 200 200 F F F 9 180-199 134 134 134 204 204 204 F F F 10 200-219 136 136 136 209 209 209 F F F 11 220-239 136 136 136 209 209 209 F F F 12 240-259 133 133 133 203 203 203 T T T 13 260-279 134 134 134 204 204 204 T T T 14 280-299 128 128 128 197 197 197 T T T 15 300-319 134 134 134 206 206 206 F F F 16 320-339 136 136 136 208 208 208 F F F 17 340-359 135 135 135 207 207 207 F F F 18 360-379 136 136 136 208 208 208 F F F 19 380-399 136 136 136 208 208 208 F F F 20 400-419 130 130 130 200 200 200 F F F 21 420-439 136 136 136 208 208 208 F F F 22 440-459 133 133 133 203 203 203 F F F 23 460-479 134 134 134 204 204 204 F F F 24 480-499 134 134 134 204 204 204 F F F 25 500-519 133 133 133 203 203 203 F F F 26 520-539 136 136 136 208 208 208 F F F 27 540-559 133 133 133 203 203 203 F F F 28 560-579 136 136 136 209 209 209 T T T 29 580-599 134 134 134 205 205 205 F F F Max gen: 136, Max flush: 209 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 203 203 203 F F F 1 020-039 125 125 125 193 193 193 F F F 2 040-059 134 134 134 204 204 204 F F F 3 060-079 135 135 135 207 207 207 F F F 4 080-099 134 134 134 205 205 205 F F F 5 100-119 136 136 136 209 209 209 F F F 6 120-139 136 136 136 209 209 209 F F F 7 140-159 129 129 129 198 198 198 F F F 8 160-179 130 130 130 200 200 200 F F F 9 180-199 134 134 134 204 204 204 F F F 10 200-219 136 136 136 209 209 209 F F F 11 220-239 136 136 136 209 209 209 F F F 12 240-259 137 137 137 210 210 210 F F F 13 260-279 137 137 137 210 210 210 F F F 14 280-299 137 137 137 210 210 210 F F F 15 300-319 134 134 134 206 206 206 F F F 16 320-339 136 136 136 208 208 208 F F F 17 340-359 135 135 135 207 207 207 F F F 18 360-379 136 136 136 208 208 208 F F F 19 380-399 136 136 136 208 208 208 F F F 20 400-419 130 130 130 200 200 200 F F F 21 420-439 136 136 136 208 208 208 F F F 22 440-459 133 133 133 203 203 203 F F F 23 460-479 134 134 134 204 204 204 F F F 24 480-499 134 134 134 204 204 204 F F F 25 500-519 133 133 133 203 203 203 F F F 26 520-539 136 136 136 208 208 208 F F F 27 540-559 133 133 133 203 203 203 F F F 28 560-579 137 137 137 210 210 210 F F F 29 580-599 134 134 134 205 205 205 F F F Max gen: 137, Max flush: 210 Begin loop 69 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.188982134Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.232654107Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.233293734Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.233330599Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.233346286Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.233361204Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739} {"msg":"Crucible 40a2b69d-5faa-4062-9ace-395dae8fdd13 has session id: 20534a22-f198-40b7-a937-7762d4407245","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.233833955Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"Upstairs opts: Upstairs UUID: 40a2b69d-5faa-4062-9ace-395dae8fdd13, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.233882168Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"Crucible stats registered with UUID: 40a2b69d-5faa-4062-9ace-395dae8fdd13","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.233906535Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.26509672Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.265133485Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739} {"msg":"40a2b69d-5faa-4062-9ace-395dae8fdd13 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.265161041Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.265177738Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"client":"0","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.265205594Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"client":"0","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.265223672Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"client":"1","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.265240729Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"client":"1","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.265257447Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"client":"2","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.265274364Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"client":"2","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.265290962Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.265315728Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.265349423Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"ds_connection connected from Ok(127.0.0.1:33003)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.265478115Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"ds_connection connected from Ok(127.0.0.1:35780)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.265519359Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"ds_connection connected from Ok(127.0.0.1:48616)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.265551054Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.266314772Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"client":"0","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.266355417Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"client":"0","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.266389441Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"client":"1","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.266416908Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"client":"1","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.266443604Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"client":"2","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.266487117Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"client":"2","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.266962618Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"client":"1","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.267004122Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"client":"0","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.267124514Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"client":"2","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.267539224Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"client":"1","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.267567269Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"client":"1","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.267584027Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.267607384Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"client":"0","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.26763461Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"client":"0","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.267661116Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.267687922Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"client":"2","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.267713738Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"client":"2","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"[0]R flush_numbers[0..12]: [203, 193, 204, 207, 205, 209, 209, 198, 200, 204, 209, 209]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.267742294Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"":"downstairs","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"[0]R generation[0..12]: [133, 125, 134, 135, 134, 136, 136, 129, 130, 134, 136, 136]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.26776853Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"":"downstairs","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.267795096Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"":"downstairs","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"[1]R flush_numbers[0..12]: [203, 193, 204, 207, 205, 209, 209, 198, 200, 204, 209, 209]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.267820163Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"":"downstairs","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"[1]R generation[0..12]: [133, 125, 134, 135, 134, 136, 136, 129, 130, 134, 136, 136]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.267843709Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"":"downstairs","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.267868696Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"":"downstairs","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"[2]R flush_numbers[0..12]: [203, 193, 204, 207, 205, 209, 209, 198, 200, 204, 209, 209]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.267896221Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"":"downstairs","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"[2]R generation[0..12]: [133, 125, 134, 135, 134, 136, 136, 129, 130, 134, 136, 136]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.267922748Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"":"downstairs","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.267949244Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"":"downstairs","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"Max found gen is 138","v":0,"name":"crucible",Wait for a query_work_queue command to finish before sending IO "level":30,"time":"2025-01-03T08:35:47.267986688Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"":"downstairs","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.268016974Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"":"downstairs","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"Next flush: 211","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.26804435Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"":"downstairs","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.268062167Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"":"downstairs","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.268075695Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.268091953Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"client":"0","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.268105601Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"client":"1","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.268118709Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"client":"2","":"downstairs","session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.268131787Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.268145145Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"40a2b69d-5faa-4062-9ace-395dae8fdd13 is now active with session: 20534a22-f198-40b7-a937-7762d4407245","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.268158123Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.268170372Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739,"session_id":"20534a22-f198-40b7-a937-7762d4407245"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:47.26818259Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 116 len 8192 data: 14 17 02/30 Read block 224 len 36864 03/30 Read block 70 len 20480 04/30 Read block 547 len 16384 05/30 Read block 228 len 20480 06/30 Read block 284 len 4096 07/30 Read block 29 len 20480 08/30 Flush 09/30 Flush 10/30 Write block 92 len 28672 data: 13 11 9 10 10 13 16 11/30 Read block 187 len 20480 12/30 Write block 80 len 36864 data: 11 10 10 10 9 17 14 12 13 13/30 Write block 487 len 20480 data: 10 7 9 9 9 14/30 Write block 188 len 8192 data: 15 13 15/30 Read block 547 len 8192 16/30 Write block 163 len 20480 data: 13 16 15 14 15 17/30 Write block 208 len 8192 data: 6 6 18/30 Read block 85 len 20480 19/30 Read block 251 len 4096 20/30 Read block 90 len 28672 21/30 Flush 22/30 Flush 23/30 Write block 241 len 16384 data: 14 13 14 16 24/30 Read block 217 len 36864 25/30 Write block 507 len 16384 data: 9 10 9 9 26/30 Read block 178 len 32768 27/30 Read block 481 len 20480 28/30 Write block 190 len 36864 data: 14 13 9 9 9 9 11 11 10 29/30 Read block 105 len 12288 30/30 Write block 543 len 12288 data: 10 11 11 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 4 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:35:47.28667259Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1739} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 203 203 203 F F F 1 020-039 125 125 125 193 193 193 F F F 2 040-059 134 134 134 204 204 204 F F F 3 060-079 135 135 135 207 207 207 F F F 4 080-099 138 138 138 213 213 213 F F F 5 100-119 138 138 138 211 211 211 F F F 6 120-139 136 136 136 209 209 209 F F F 7 140-159 129 129 129 198 198 198 F F F 8 160-179 138 138 138 213 213 213 F F F 9 180-199 138 138 138 213 213 213 T T T 10 200-219 138 138 138 213 213 213 F F F 11 220-239 136 136 136 209 209 209 F F F 12 240-259 137 137 137 210 210 210 T T T 13 260-279 137 137 137 210 210 210 F F F 14 280-299 137 137 137 210 210 210 F F F 15 300-319 134 134 134 206 206 206 F F F 16 320-339 136 136 136 208 208 208 F F F 17 340-359 135 135 135 207 207 207 F F F 18 360-379 136 136 136 208 208 208 F F F 19 380-399 136 136 136 208 208 208 F F F 20 400-419 130 130 130 200 200 200 F F F 21 420-439 136 136 136 208 208 208 F F F 22 440-459 133 133 133 203 203 203 F F F 23 460-479 134 134 134 204 204 204 F F F 24 480-499 138 138 138 213 213 213 F F F 25 500-519 133 133 133 203 203 203 T T T 26 520-539 136 136 136 208 208 208 F F F 27 540-559 133 133 133 203 203 203 T T T 28 560-579 137 137 137 210 210 210 F F F 29 580-599 134 134 134 205 205 205 F F F Max gen: 138, Max flush: 213 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 203 203 203 F F F 1 020-039 125 125 125 193 193 193 F F F 2 040-059 134 134 134 204 204 204 F F F 3 060-079 135 135 135 207 207 207 F F F 4 080-099 138 138 138 213 213 213 F F F 5 100-119 138 138 138 211 211 211 F F F 6 120-139 136 136 136 209 209 209 F F F 7 140-159 129 129 129 198 198 198 F F F 8 160-179 138 138 138 213 213 213 F F F 9 180-199 139 139 139 214 214 214 F F F 10 200-219 138 138 138 213 213 213 F F F 11 220-239 136 136 136 209 209 209 F F F 12 240-259 139 139 139 214 214 214 F F F 13 260-279 137 137 137 210 210 210 F F F 14 280-299 137 137 137 210 210 210 F F F 15 300-319 134 134 134 206 206 206 F F F 16 320-339 136 136 136 208 208 208 F F F 17 340-359 135 135 135 207 207 207 F F F 18 360-379 136 136 136 208 208 208 F F F 19 380-399 136 136 136 208 208 208 F F F 20 400-419 130 130 130 200 200 200 F F F 21 420-439 136 136 136 208 208 208 F F F 22 440-459 133 133 133 203 203 203 F F F 23 460-479 134 134 134 204 204 204 F F F 24 480-499 138 138 138 213 213 213 F F F 25 500-519 139 139 139 214 214 214 F F F 26 520-539 136 136 136 208 208 208 F F F 27 540-559 139 139 139 214 214 214 F F F 28 560-579 137 137 137 210 210 210 F F F 29 580-599 134 134 134 205 205 205 F F F Max gen: 139, Max flush: 214 Begin loop 70 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.546370635Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.590133435Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.590801837Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.590841571Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.590859539Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.590877646Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747} {"msg":"Crucible 4a3d2ff3-2581-4dee-b10c-d14ae9629973 has session id: 1a52c47d-6ebf-483c-b348-e93934c85386","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.591371464Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"Upstairs opts: Upstairs UUID: 4a3d2ff3-2581-4dee-b10c-d14ae9629973, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.591414218Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"Crucible stats registered with UUID: 4a3d2ff3-2581-4dee-b10c-d14ae9629973","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.591438424Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.622183725Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.622224049Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747} {"msg":"4a3d2ff3-2581-4dee-b10c-d14ae9629973 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.622241637Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.622258594Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"client":"0","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.62228734Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"client":"0","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.622311747Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"client":"1","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.622329854Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"client":"1","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.622346631Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"client":"2","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.622364229Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"client":"2","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.622394934Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.622430419Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.622464274Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"ds_connection connected from Ok(127.0.0.1:45691)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.622499059Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"ds_connection connected from Ok(127.0.0.1:48192)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.622533734Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"ds_connection connected from Ok(127.0.0.1:55685)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.622760311Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.623249879Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"client":"0","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.623285004Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"client":"0","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.623303312Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"client":"1","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.623320759Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"client":"1","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.62338511Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"client":"2","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.623409756Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"client":"2","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.623962905Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"client":"1","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.623994551Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"client":"0","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.624123632Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"client":"2","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.624639476Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"client":"1","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.624673682Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"client":"1","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.624689359Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.624701858Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"client":"0","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.624715285Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"client":"0","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.624728794Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.624741252Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"client":"2","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.6247547Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"client":"2","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"[0]R flush_numbers[0..12]: [203, 193, 204, 207, 213, 211, 209, 198, 213, 214, 213, 209]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.624768878Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"":"downstairs","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"[0]R generation[0..12]: [133, 125, 134, 135, 138, 138, 136, 129, 138, 139, 138, 136]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.624783126Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"":"downstairs","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.624797064Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"":"downstairs","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"[1]R flush_numbers[0..12]: [203, 193, 204, 207, 213, 211, 209, 198, 213, 214, 213, 209]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.624811062Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"":"downstairs","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"[1]R generation[0..12]: [133, 125, 134, 135, 138, 138, 136, 129, 138, 139, 138, 136]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.624829049Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"":"downstairs","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.624843937Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"":"downstairs","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"[2]R flush_numbers[0..12]: [203, 193, 204, 207, 213, 211, 209, 198, 213, 214, 213, 209]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.624857705Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"":"downstairs","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"[2]R generation[0..12]: [133, 125, 134, 135, 138, 138, 136, 129, 138, 139, 138, 136]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.624871523Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"":"downstairs","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]Wait for a query_work_queue command to finish before sending IO ","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.624894169Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"":"downstairs","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.624915476Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"":"downstairs","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.624940443Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"":"downstairs","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"Next flush: 215","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.624955381Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"":"downstairs","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.624968748Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"":"downstairs","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.624982057Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.625001254Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"client":"0","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.62502506Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"client":"1","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.625039838Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"client":"2","":"downstairs","session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.625053566Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.625069634Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"4a3d2ff3-2581-4dee-b10c-d14ae9629973 is now active with session: 1a52c47d-6ebf-483c-b348-e93934c85386","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.625088411Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-03T08:35:51.625111978Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747,"session_id":"1a52c47d-6ebf-483c-b348-e93934c85386"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:51.625129055Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 419 len 20480 data: 17 13 15 12 10 02/30 Read block 581 len 24576 03/30 Read block 140 len 32768 04/30 Read block 439 len 40960 05/30 Write block 144 len 24576 data: 12 14 15 16 12 13 06/30 Write block 70 len 12288 data: 13 11 13 07/30 Read block 254 len 8192 08/30 Write block 329 len 24576 data: 8 9 7 6 7 8 09/30 Read block 68 len 40960 10/30 Read block 447 len 8192 11/30 Write block 322 len 20480 data: 10 11 11 10 11 12/30 Read block 316 len 28672 13/30 Read block 51 len 4096 14/30 Write block 127 len 40960 data: 12 11 15 15 16 14 11 10 10 7 15/30 Flush 16/30 Write block 252 len 40960 data: 16 16 16 13 17 16 16 15 14 14 17/30 Write block 476 len 20480 data: 14 12 10 9 11 18/30 Write block 39 len 24576 data: 11 11 10 13 11 11 19/30 Read block 454 len 36864 20/30 Flush 21/30 Write block 417 len 24576 data: 16 14 18 14 16 13 22/30 Write block 372 len 20480 data: 14 12 11 11 12 23/30 Write block 394 len 36864 data: 12 10 10 12 13 13 12 11 10 24/30 Write block 471 len 20480 data: 13 13 14 10 10 25/30 Write block 74 len 32768 data: 15 15 16 15 12 11 12 11 26/30 Read block 90 len 20480 27/30 Read block 164 len 40960 28/30 Write block 253 len 8192 data: 17 17 29/30 Write block 349 len 24576 data: 11 13 13 12 11 10 30/30 Read block 241 len 32768 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 6 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 6 Done Sent Done false 1029 Acked Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 0 3 Done 9 8 10 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:35:51.644669082Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1747} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 203 203 203 F F F 1 020-039 140 140 140 216 216 216 F F F 2 040-059 140 140 140 216 216 216 F F F 3 060-079 140 140 140 215 215 215 T T T 4 080-099 138 138 138 213 213 213 T T T 5 100-119 138 138 138 211 211 211 F F F 6 120-139 140 140 140 215 215 215 F F F 7 140-159 140 140 140 215 215 215 F F F 8 160-179 138 138 138 213 213 213 F F F 9 180-199 139 139 139 214 214 214 F F F 10 200-219 138 138 138 213 213 213 F F F 11 220-239 136 136 136 209 209 209 F F F 12 240-259 140 140 140 216 216 216 T T T 13 260-279 140 140 140 216 216 216 F F F 14 280-299 137 137 137 210 210 210 F F F 15 300-319 134 134 134 206 206 206 F F F 16 320-339 140 140 140 215 215 215 F F F 17 340-359 135 135 135 207 207 207 T T T 18 360-379 136 136 136 208 208 208 T T T 19 380-399 136 136 136 208 208 208 T T T 20 400-419 140 140 140 215 215 215 T T T 21 420-439 140 140 140 215 215 215 T T T 22 440-459 133 133 133 203 203 203 F F F 23 460-479 140 140 140 216 216 216 T T T 24 480-499 140 140 140 216 216 216 F F F 25 500-519 139 139 139 214 214 214 F F F 26 520-539 136 136 136 208 208 208 F F F 27 540-559 139 139 139 214 214 214 F F F 28 560-579 137 137 137 210 210 210 F F F 29 580-599 134 134 134 205 205 205 F F F Max gen: 140, Max flush: 216 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 203 203 203 F F F 1 020-039 140 140 140 216 216 216 F F F 2 040-059 140 140 140 216 216 216 F F F 3 060-079 141 141 141 217 217 217 F F F 4 080-099 141 141 141 217 217 217 F F F 5 100-119 138 138 138 211 211 211 F F F 6 120-139 140 140 140 215 215 215 F F F 7 140-159 140 140 140 215 215 215 F F F 8 160-179 138 138 138 213 213 213 F F F 9 180-199 139 139 139 214 214 214 F F F 10 200-219 138 138 138 213 213 213 F F F 11 220-239 136 136 136 209 209 209 F F F 12 240-259 141 141 141 217 217 217 F F F 13 260-279 140 140 140 216 216 216 F F F 14 280-299 137 137 137 210 210 210 F F F 15 300-319 134 134 134 206 206 206 F F F 16 320-339 140 140 140 215 215 215 F F F 17 340-359 141 141 141 217 217 217 F F F 18 360-379 141 141 141 217 217 217 F F F 19 380-399 141 141 141 217 217 217 F F F 20 400-419 141 141 141 217 217 217 F F F 21 420-439 141 141 141 217 217 217 F F F 22 440-459 133 133 133 203 203 203 F F F 23 460-479 141 141 141 217 217 217 F F F 24 480-499 140 140 140 216 216 216 F F F 25 500-519 139 139 139 214 214 214 F F F 26 520-539 136 136 136 208 208 208 F F F 27 540-559 139 139 139 214 214 214 F F F 28 560-579 137 137 137 210 210 210 F F F 29 580-599 134 134 134 205 205 205 F F F Max gen: 141, Max flush: 217 Begin loop 71 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:55.941909255Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:35:55.984436965Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:55.98481643Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.016843243Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.017431147Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.017466992Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.01748148Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.017496238Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755} {"msg":"Crucible 1bc5655e-802d-46c7-9f18-195bd2fd33e9 has session id: c6478954-115f-415e-8bb6-8c9f036625d5","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.017903149Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"Upstairs opts: Upstairs UUID: 1bc5655e-802d-46c7-9f18-195bd2fd33e9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.017946332Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"Crucible stats registered with UUID: 1bc5655e-802d-46c7-9f18-195bd2fd33e9","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.017971828Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.048655608Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.048685014Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755} {"msg":"1bc5655e-802d-46c7-9f18-195bd2fd33e9 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.048720159Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.048737316Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"client":"0","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.048763363Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"client":"0","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.04878251Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"client":"1","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.048804207Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"client":"1","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.048821614Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"client":"2","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.048838632Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"client":"2","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.048855229Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"":"io task","client":"2","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.048872876Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"":"io task","client":"0","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.048903322Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"":"io task","client":"1","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:37106)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.049000218Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"":"io task","client":"1","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:48264)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.049059269Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"":"io task","client":"2","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:35611)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.049215397Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"":"io task","client":"0","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.04981036Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"client":"1","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.049851184Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"client":"1","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.049882519Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"client":"2","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.049912015Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"client":"2","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.04994338Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"client":"0","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.049962388Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"client":"0","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.050460825Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"client":"2","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.050511217Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"client":"1","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.050637489Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"client":"0","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.051079334Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"client":"1","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.051119699Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"client":"1","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.051149284Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.05117657Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"client":"2","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.051203196Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"client":"2","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.051227403Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.0512493Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"client":"0","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.051273376Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"client":"0","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"[0]R flush_numbers[0..12]: [203, 216, 216, 217, 217, 211, 215, 215, 213, 214, 213, 209]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.051299762Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"":"downstairs","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"[0]R generation[0..12]: [133, 140, 140, 141, 141, 138, 140, 140, 138, 139, 138, 136]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.051324459Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"":"downstairs","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.051348665Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"":"downstairs","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"[1]R flush_numbers[0..12]: [203, 216, 216, 217, 217, 211, 215, 215, 213, 214, 213, 209]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.051375711Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"":"downstairs","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"[1]R generation[0..12]: [133, 140, 140, 141, 141, 138, 140, 140, 138, 139, 138, 136]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.051407737Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"":"downstairs","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.051434842Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"":"downstairs","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"[2]R flush_numbers[0..12]: [203, 216, 216, 217, 217, 211, 215, 215, 213, 214, 213, 209]","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2025-01-03T08:35:56.051472887Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"":"downstairs","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"[2]R generation[0..12]: [133, 140, 140, 141, 141, 138, 140, 140, 138, 139, 138, 136]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.051504722Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"":"downstairs","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.051535178Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"":"downstairs","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.051564583Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"":"downstairs","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.05159259Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"":"downstairs","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"Next flush: 218","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.051622465Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"":"downstairs","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.051650771Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"":"downstairs","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"No downstairs reconciliation required"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.051682636Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.051706003Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"client":"0","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.051720771Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"client":"1","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.051734359Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"client":"2","":"downstairs","session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.051748007Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.051763574Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"1bc5655e-802d-46c7-9f18-195bd2fd33e9 is now active with session: c6478954-115f-415e-8bb6-8c9f036625d5","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.051777112Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.051789321Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755,"session_id":"c6478954-115f-415e-8bb6-8c9f036625d5"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:35:56.051801999Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 311 len 8192 data: 10 11 02/30 Read block 472 len 32768 03/30 Write block 162 len 16384 data: 15 14 17 16 04/30 Write block 539 len 32768 data: 4 5 7 10 11 12 12 13 05/30 Flush 06/30 Flush 07/30 Write block 5 len 8192 data: 3 3 08/30 Read block 497 len 8192 09/30 Read block 520 len 40960 10/30 Write block 309 len 12288 data: 8 9 11 11/30 Write block 493 len 12288 data: 13 12 13 12/30 Flush 13/30 Write block 119 len 28672 data: 19 16 14 14 17 14 12 14/30 Read block 401 len 16384 15/30 Write block 103 len 24576 data: 14 12 10 12 10 11 16/30 Write block 460 len 8192 data: 13 13 17/30 Read block 486 len 24576 18/30 Write block 214 len 24576 data: 11 9 10 10 10 10 19/30 Write block 265 len 16384 data: 11 14 14 15 20/30 Flush 21/30 Write block 194 len 36864 data: 10 10 12 12 11 11 11 10 10 22/30 Write block 375 len 28672 data: 12 13 14 12 10 9 7 23/30 Write block 243 len 28672 data: 15 17 17 15 16 15 13 24/30 Write block 564 len 36864 data: 11 10 11 11 12 12 11 11 11 25/30 Read block 348 len 20480 26/30 Write block 275 len 16384 data: 10 9 9 8 27/30 Read block 464 len 40960 28/30 Write block 434 len 12288 data: 5 9 10 29/30 Read block 330 len 36864 30/30 Read block 219 len 16384 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 9 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 9 10 9 28 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:35:56.070111775Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1755} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 220 220 220 F F F 1 020-039 140 140 140 216 216 216 F F F 2 040-059 140 140 140 216 216 216 F F F 3 060-079 141 141 141 217 217 217 F F F 4 080-099 141 141 141 217 217 217 F F F 5 100-119 142 142 142 221 221 221 F F F 6 120-139 142 142 142 221 221 221 F F F 7 140-159 140 140 140 215 215 215 F F F 8 160-179 142 142 142 218 218 218 F F F 9 180-199 139 139 139 214 214 214 T T T 10 200-219 142 142 142 221 221 221 T T T 11 220-239 136 136 136 209 209 209 F F F 12 240-259 141 141 141 217 217 217 T T T 13 260-279 142 142 142 221 221 221 T T T 14 280-299 137 137 137 210 210 210 F F F 15 300-319 142 142 142 220 220 220 F F F 16 320-339 140 140 140 215 215 215 F F F 17 340-359 141 141 141 217 217 217 F F F 18 360-379 141 141 141 217 217 217 T T T 19 380-399 141 141 141 217 217 217 T T T 20 400-419 141 141 141 217 217 217 F F F 21 420-439 141 141 141 217 217 217 T T T 22 440-459 133 133 133 203 203 203 F F F 23 460-479 142 142 142 221 221 221 F F F 24 480-499 142 142 142 220 220 220 F F F 25 500-519 139 139 139 214 214 214 F F F 26 520-539 142 142 142 218 218 218 F F F 27 540-559 142 142 142 218 218 218 F F F 28 560-579 137 137 137 210 210 210 T T T 29 580-599 134 134 134 205 205 205 F F F Max gen: 142, Max flush: 221 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 220 220 220 F F F 1 020-039 140 140 140 216 216 216 F F F 2 040-059 140 140 140 216 216 216 F F F 3 060-079 141 141 141 217 217 217 F F F 4 080-099 141 141 141 217 217 217 F F F 5 100-119 142 142 142 221 221 221 F F F 6 120-139 142 142 142 221 221 221 F F F 7 140-159 140 140 140 215 215 215 F F F 8 160-179 142 142 142 218 218 218 F F F 9 180-199 143 143 143 222 222 222 F F F 10 200-219 143 143 143 222 222 222 F F F 11 220-239 136 136 136 209 209 209 F F F 12 240-259 143 143 143 222 222 222 F F F 13 260-279 143 143 143 222 222 222 F F F 14 280-299 137 137 137 210 210 210 F F F 15 300-319 142 142 142 220 220 220 F F F 16 320-339 140 140 140 215 215 215 F F F 17 340-359 141 141 141 217 217 217 F F F 18 360-379 143 143 143 222 222 222 F F F 19 380-399 143 143 143 222 222 222 F F F 20 400-419 141 141 141 217 217 217 F F F 21 420-439 143 143 143 222 222 222 F F F 22 440-459 133 133 133 203 203 203 F F F 23 460-479 142 142 142 221 221 221 F F F 24 480-499 142 142 142 220 220 220 F F F 25 500-519 139 139 139 214 214 214 F F F 26 520-539 142 142 142 218 218 218 F F F 27 540-559 142 142 142 218 218 218 F F F 28 560-579 143 143 143 222 222 222 F F F 29 580-599 134 134 134 205 205 205 F F F Max gen: 143, Max flush: 222 Begin loop 72 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.383412483Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.427114912Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.427727282Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.427766447Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.427785784Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.427800902Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763} {"msg":"Crucible e17141e9-e369-4a25-9ea7-e7a511a1c311 has session id: 4042994b-540d-4554-aeb6-02eb3f87aa69","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.428274633Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"Upstairs opts: Upstairs UUID: e17141e9-e369-4a25-9ea7-e7a511a1c311, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.428379347Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"Crucible stats registered with UUID: e17141e9-e369-4a25-9ea7-e7a511a1c311","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.428402434Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.45911561Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.459149954Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763} {"msg":"e17141e9-e369-4a25-9ea7-e7a511a1c311 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.45918006Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.459197727Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"client":"0","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.459225833Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"client":"0","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.459244111Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"client":"1","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.459261738Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"client":"1","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.459279006Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"client":"2","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.459301822Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"client":"2","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.45932048Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"":"io task","client":"2","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.459351345Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"":"io task","client":"0","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.45938785Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"":"io task","client":"1","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"ds_connection connected from Ok(127.0.0.1:39057)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.459485705Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"":"io task","client":"2","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"ds_connection connected from Ok(127.0.0.1:51317)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.459536698Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"":"io task","client":"0","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"ds_connection connected from Ok(127.0.0.1:53740)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.459610957Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"":"io task","client":"1","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.460243765Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"client":"2","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.4602738Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"client":"2","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.460445835Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"client":"1","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.460487599Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"client":"1","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.460516845Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"client":"0","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.460544731Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"client":"0","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.460938893Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"client":"2","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.460972128Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"client":"0","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.461049587Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"client":"1","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.46150938Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"client":"0","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.461550784Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"client":"0","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.46158079Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.461650579Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"client":"1","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.461679155Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"client":"1","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.461694683Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.461707651Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"client":"2","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.461721709Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"client":"2","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"[0]R flush_numbers[0..12]: [220, 216, 216, 217, 217, 221, 221, 215, 218, 222, 222, 209]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.461735907Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"":"downstairs","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"[0]R generation[0..12]: [142, 140, 140, 141, 141, 142, 142, 140, 142, 143, 143, 136]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.461749955Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"":"downstairs","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.461764083Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"":"downstairs","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"[1]R flush_numbers[0..12]: [220, 216, 216, 217, 217, 221, 221, 215, 218, 222, 222, 209]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.461778081Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"":"downstairs","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"[1]R generation[0..12]: [142, 140, 140, 141, 141, 142, 142, 140, 142, 143, 143, 136]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.461791979Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"":"downstairs","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.461806157Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"":"downstairs","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"[2]R flush_numbers[0..12]: [220, 216, 216, 217, 217, 221, 221, 215, 218, 222, 222, 209]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.461824044Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"":"downstairs","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"[2]R generation[0..12]: [142, 140, 140, 141, 141, 142, 142, 140, 142, 143, 143, 136]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.461838432Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"":"downstairs","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.46185248Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"":"downstairs","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.461865868Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"":"downstairs","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.461879376Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"":"downstairs","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"Next flush: 223","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:36:00.461892734Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"":"downstairs","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.461915091Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"":"downstairs","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.461928779Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.461941847Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"client":"0","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.461955505Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"client":"1","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.461968993Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"client":"2","":"downstairs","session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.461982641Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.461995129Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"e17141e9-e369-4a25-9ea7-e7a511a1c311 is now active with session: 4042994b-540d-4554-aeb6-02eb3f87aa69","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.462008387Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.462020945Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763,"session_id":"4042994b-540d-4554-aeb6-02eb3f87aa69"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:00.462035303Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 547 len 32768 data: 13 16 13 15 16 15 13 11 02/30 Read block 480 len 12288 03/30 Write block 348 len 8192 data: 11 12 04/30 Write block 508 len 4096 data: 11 05/30 Read block 389 len 32768 06/30 Write block 325 len 20480 data: 11 12 10 8 9 07/30 Read block 383 len 28672 08/30 Read block 574 len 36864 09/30 Read block 477 len 8192 10/30 Read block 173 len 20480 11/30 Write block 539 len 40960 data: 5 6 8 11 12 13 13 14 14 17 12/30 Read block 387 len 28672 13/30 Flush 14/30 Write block 518 len 28672 data: 10 10 11 12 11 12 13 15/30 Write block 409 len 16384 data: 13 15 17 17 16/30 Write block 14 len 12288 data: 11 13 13 17/30 Read block 306 len 8192 18/30 Read block 233 len 28672 19/30 Write block 160 len 16384 data: 12 14 16 15 20/30 Write block 272 len 16384 data: 10 11 10 11 21/30 Write block 167 len 4096 data: 16 22/30 Read block 116 len 32768 23/30 Write block 81 len 24576 data: 12 11 11 10 18 15 24/30 Read block 485 len 32768 25/30 Read block 101 len 12288 26/30 Flush 27/30 Write block 376 len 24576 data: 14 15 13 11 10 8 28/30 Write block 142 len 12288 data: 11 12 13 29/30 Write block 199 len 4096 data: 12 30/30 Read block 455 len 8192 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 6 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:36:00.480451414Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1763} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 224 224 224 F F F 1 020-039 140 140 140 216 216 216 F F F 2 040-059 140 140 140 216 216 216 F F F 3 060-079 141 141 141 217 217 217 F F F 4 080-099 144 144 144 224 224 224 F F F 5 100-119 142 142 142 221 221 221 F F F 6 120-139 142 142 142 221 221 221 F F F 7 140-159 140 140 140 215 215 215 T T T 8 160-179 144 144 144 224 224 224 F F F 9 180-199 143 143 143 222 222 222 T T T 10 200-219 143 143 143 222 222 222 F F F 11 220-239 136 136 136 209 209 209 F F F 12 240-259 143 143 143 222 222 222 F F F 13 260-279 144 144 144 224 224 224 F F F 14 280-299 137 137 137 210 210 210 F F F 15 300-319 142 142 142 220 220 220 F F F 16 320-339 144 144 144 223 223 223 F F F 17 340-359 144 144 144 223 223 223 F F F 18 360-379 143 143 143 222 222 222 T T T 19 380-399 143 143 143 222 222 222 T T T 20 400-419 144 144 144 224 224 224 F F F 21 420-439 143 143 143 222 222 222 F F F 22 440-459 133 133 133 203 203 203 F F F 23 460-479 142 142 142 221 221 221 F F F 24 480-499 142 142 142 220 220 220 F F F 25 500-519 144 144 144 224 224 224 F F F 26 520-539 144 144 144 224 224 224 F F F 27 540-559 144 144 144 223 223 223 F F F 28 560-579 143 143 143 222 222 222 F F F 29 580-599 134 134 134 205 205 205 F F F Max gen: 144, Max flush: 224 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 224 224 224 F F F 1 020-039 140 140 140 216 216 216 F F F 2 040-059 140 140 140 216 216 216 F F F 3 060-079 141 141 141 217 217 217 F F F 4 080-099 144 144 144 224 224 224 F F F 5 100-119 142 142 142 221 221 221 F F F 6 120-139 142 142 142 221 221 221 F F F 7 140-159 145 145 145 225 225 225 F F F 8 160-179 144 144 144 224 224 224 F F F 9 180-199 145 145 145 225 225 225 F F F 10 200-219 143 143 143 222 222 222 F F F 11 220-239 136 136 136 209 209 209 F F F 12 240-259 143 143 143 222 222 222 F F F 13 260-279 144 144 144 224 224 224 F F F 14 280-299 137 137 137 210 210 210 F F F 15 300-319 142 142 142 220 220 220 F F F 16 320-339 144 144 144 223 223 223 F F F 17 340-359 144 144 144 223 223 223 F F F 18 360-379 145 145 145 225 225 225 F F F 19 380-399 145 145 145 225 225 225 F F F 20 400-419 144 144 144 224 224 224 F F F 21 420-439 143 143 143 222 222 222 F F F 22 440-459 133 133 133 203 203 203 F F F 23 460-479 142 142 142 221 221 221 F F F 24 480-499 142 142 142 220 220 220 F F F 25 500-519 144 144 144 224 224 224 F F F 26 520-539 144 144 144 224 224 224 F F F 27 540-559 144 144 144 223 223 223 F F F 28 560-579 143 143 143 222 222 222 F F F 29 580-599 134 134 134 205 205 205 F F F Max gen: 145, Max flush: 225 Begin loop 73 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.739802719Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:36:04.781709279Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.782070906Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.813496008Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.814120317Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.814160971Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.814182668Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.814201305Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771} {"msg":"Crucible 36e3de70-dfd9-4793-b997-9f1376abf8d8 has session id: 06c3c64d-2f00-40ce-a616-7d5e0d8f9401","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.81464429Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"Upstairs opts: Upstairs UUID: 36e3de70-dfd9-4793-b997-9f1376abf8d8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.814687594Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"Crucible stats registered with UUID: 36e3de70-dfd9-4793-b997-9f1376abf8d8","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.81471193Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.844997368Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.845042751Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771} {"msg":"36e3de70-dfd9-4793-b997-9f1376abf8d8 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.845108542Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.845136718Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"client":"0","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.845162044Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"client":"0","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.845177452Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"client":"1","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.845195849Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"client":"1","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.845221855Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"client":"2","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.845248021Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"client":"2","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.845270248Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"":"io task","client":"2","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.845290445Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"":"io task","client":"0","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.845315832Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"ds_connection connected from Ok(127.0.0.1:44829)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.845337368Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"":"io task","client":"2","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"ds_connection connected from Ok(127.0.0.1:42436)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.845495496Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"":"io task","client":"0","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"ds_connection connected from Ok(127.0.0.1:45064)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.845565145Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.846073051Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"client":"2","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.846102357Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"client":"2","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.846282501Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"client":"1","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.846317805Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"client":"1","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.846347251Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"client":"0","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.846382886Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"client":"0","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.846838779Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"client":"2","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.846880803Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"client":"1","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.84710727Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"client":"0","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.847534768Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"client":"2","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.847566003Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"client":"2","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.847581571Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.847594619Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"client":"1","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.847608187Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"client":"1","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.847621345Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.847633633Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"client":"0","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.847646991Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"client":"0","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"[0]R flush_numbers[0..12]: [224, 216, 216, 217, 224, 221, 221, 225, 224, 225, 222, 209]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.847660809Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"":"downstairs","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"[0]R generation[0..12]: [144, 140, 140, 141, 144, 142, 142, 145, 144, 145, 143, 136]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.847674977Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"":"downstairs","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.847689245Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"":"downstairs","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"[1]R flush_numbers[0..12]: [224, 216, 216, 217, 224, 221, 221, 225, 224, 225, 222, 209]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.847703133Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"":"downstairs","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"[1]R generation[0..12]: [144, 140, 140, 141, 144, 142, 142, 145, 144, 145, 143, 136]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.847716921Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"":"downstairs","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.847731109Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"":"downstairs","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"[2]R flush_numbers[0..12]: [224, 216, 216, 217, 224, 221, 221, 225, 224, 225, 222, 209]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.847744877Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"":"downstairs","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"[2]R generation[0..12]: [144, 140, 140, 141, 144, 142, 142, 145, 144, 145, 143, 136]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.847758715Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"":"downstairs","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:36:04.847774623Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"":"downstairs","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.847846632Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"":"downstairs","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.847870998Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"":"downstairs","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"Next flush: 226","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.847886066Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"":"downstairs","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.847900134Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"":"downstairs","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.847914882Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.84792792Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"client":"0","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.847941998Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"client":"1","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.847956376Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"client":"2","":"downstairs","session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.847970614Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.847983982Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"36e3de70-dfd9-4793-b997-9f1376abf8d8 is now active with session: 06c3c64d-2f00-40ce-a616-7d5e0d8f9401","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.84799766Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.848010028Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771,"session_id":"06c3c64d-2f00-40ce-a616-7d5e0d8f9401"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:04.848023436Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 160 len 32768 data: 13 15 17 16 18 17 15 17 02/30 Read block 30 len 36864 03/30 Read block 593 len 24576 04/30 Read block 110 len 28672 05/30 Read block 268 len 16384 06/30 Read block 395 len 40960 07/30 Read block 320 len 16384 08/30 Read block 411 len 40960 09/30 Read block 92 len 4096 10/30 Read block 140 len 12288 11/30 Write block 219 len 16384 data: 11 10 12 11 12/30 Read block 73 len 32768 13/30 Read block 350 len 16384 14/30 Read block 277 len 32768 15/30 Read block 49 len 4096 16/30 Write block 183 len 40960 data: 15 17 16 19 16 16 14 15 14 10 17/30 Write block 459 len 12288 data: 13 14 14 18/30 Read block 242 len 40960 19/30 Read block 49 len 24576 20/30 Read block 502 len 28672 21/30 Flush 22/30 Write block 364 len 8192 data: 11 14 23/30 Write block 260 len 28672 data: 15 15 15 15 14 12 15 24/30 Write block 441 len 36864 data: 11 12 12 12 14 13 12 13 12 25/30 Write block 553 len 24576 data: 14 12 11 8 7 8 26/30 Write block 199 len 32768 data: 13 12 11 11 8 6 6 7 27/30 Read block 275 len 40960 28/30 Write block 545 len 32768 data: 14 15 15 18 14 16 17 16 29/30 Read block 212 len 32768 30/30 Write block 143 len 28672 data: 13 14 15 16 17 13 14 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 2 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:36:04.868856085Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1771} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 224 224 224 F F F 1 020-039 140 140 140 216 216 216 F F F 2 040-059 140 140 140 216 216 216 F F F 3 060-079 141 141 141 217 217 217 F F F 4 080-099 144 144 144 224 224 224 F F F 5 100-119 142 142 142 221 221 221 F F F 6 120-139 142 142 142 221 221 221 F F F 7 140-159 145 145 145 225 225 225 T T T 8 160-179 146 146 146 226 226 226 F F F 9 180-199 146 146 146 226 226 226 T T T 10 200-219 146 146 146 226 226 226 T T T 11 220-239 146 146 146 226 226 226 F F F 12 240-259 143 143 143 222 222 222 F F F 13 260-279 144 144 144 224 224 224 T T T 14 280-299 137 137 137 210 210 210 F F F 15 300-319 142 142 142 220 220 220 F F F 16 320-339 144 144 144 223 223 223 F F F 17 340-359 144 144 144 223 223 223 F F F 18 360-379 145 145 145 225 225 225 T T T 19 380-399 145 145 145 225 225 225 F F F 20 400-419 144 144 144 224 224 224 F F F 21 420-439 143 143 143 222 222 222 F F F 22 440-459 146 146 146 226 226 226 T T T 23 460-479 146 146 146 226 226 226 F F F 24 480-499 142 142 142 220 220 220 F F F 25 500-519 144 144 144 224 224 224 F F F 26 520-539 144 144 144 224 224 224 F F F 27 540-559 144 144 144 223 223 223 T T T 28 560-579 143 143 143 222 222 222 F F F 29 580-599 134 134 134 205 205 205 F F F Max gen: 146, Max flush: 226 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 224 224 224 F F F 1 020-039 140 140 140 216 216 216 F F F 2 040-059 140 140 140 216 216 216 F F F 3 060-079 141 141 141 217 217 217 F F F 4 080-099 144 144 144 224 224 224 F F F 5 100-119 142 142 142 221 221 221 F F F 6 120-139 142 142 142 221 221 221 F F F 7 140-159 147 147 147 227 227 227 F F F 8 160-179 146 146 146 226 226 226 F F F 9 180-199 147 147 147 227 227 227 F F F 10 200-219 147 147 147 227 227 227 F F F 11 220-239 146 146 146 226 226 226 F F F 12 240-259 143 143 143 222 222 222 F F F 13 260-279 147 147 147 227 227 227 F F F 14 280-299 137 137 137 210 210 210 F F F 15 300-319 142 142 142 220 220 220 F F F 16 320-339 144 144 144 223 223 223 F F F 17 340-359 144 144 144 223 223 223 F F F 18 360-379 147 147 147 227 227 227 F F F 19 380-399 145 145 145 225 225 225 F F F 20 400-419 144 144 144 224 224 224 F F F 21 420-439 143 143 143 222 222 222 F F F 22 440-459 147 147 147 227 227 227 F F F 23 460-479 146 146 146 226 226 226 F F F 24 480-499 142 142 142 220 220 220 F F F 25 500-519 144 144 144 224 224 224 F F F 26 520-539 144 144 144 224 224 224 F F F 27 540-559 147 147 147 227 227 227 F F F 28 560-579 143 143 143 222 222 222 F F F 29 580-599 134 134 134 205 205 205 F F F Max gen: 147, Max flush: 227 Begin loop 74 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.180654883Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:36:09.223077488Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.223442685Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.254883934Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.255431724Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.255467689Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.255482157Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.255497275Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779} {"msg":"Crucible 6e503189-be8d-432e-aa5f-c5e15c694320 has session id: 9a830812-038c-4884-8d52-c4f7d006c787","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.255915734Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"Upstairs opts: Upstairs UUID: 6e503189-be8d-432e-aa5f-c5e15c694320, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.255974425Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"Crucible stats registered with UUID: 6e503189-be8d-432e-aa5f-c5e15c694320","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.25601084Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.286735183Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.286782387Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779} {"msg":"6e503189-be8d-432e-aa5f-c5e15c694320 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.286864715Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.28689769Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"client":"0","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.286941053Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"client":"0","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.286969819Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"client":"1","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.286997765Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"client":"1","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.287025401Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"client":"2","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.287052567Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"client":"2","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.287079083Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"":"io task","client":"2","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.287107309Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"":"io task","client":"0","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.287153242Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"":"io task","client":"1","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"ds_connection connected from Ok(127.0.0.1:50151)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.287340175Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"":"io task","client":"2","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"ds_connection connected from Ok(127.0.0.1:42967)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.287420574Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"":"io task","client":"0","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"ds_connection connected from Ok(127.0.0.1:34064)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.287874707Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"":"io task","client":"1","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.288424167Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"client":"1","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.288452793Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"client":"1","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.288468341Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"client":"0","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.288483048Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"client":"0","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.288501625Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"client":"2","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.288519163Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"client":"2","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.289119685Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"client":"2","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.289149041Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"client":"0","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.289164809Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"client":"1","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.289748854Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"client":"1","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.289780659Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"client":"1","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.289796867Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.289813364Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"client":"2","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.289827592Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"client":"2","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.28984117Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.289854188Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"client":"0","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.289867996Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"client":"0","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"[0]R flush_numbers[0..12]: [224, 216, 216, 217, 224, 221, 221, 227, 226, 227, 227, 226]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.289882134Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"":"downstairs","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"[0]R generation[0..12]: [144, 140, 140, 141, 144, 142, 142, 147, 146, 147, 147, 146]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.289896292Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"":"downstairs","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.2899104Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"":"downstairs","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"[1]R flush_numbers[0..12]: [224, 216, 216, 217, 224, 221, 221, 227, 226, 227, 227, 226]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.289924368Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"":"downstairs","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"[1]R generation[0..12]: [144, 140, 140, 141, 144, 142, 142, 147, 146, 147, 147, 146]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.289938246Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"":"downstairs","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.289952324Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"":"downstairs","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"[2]R flush_numbers[0..12]: [224, 216, 216, 217, 224, 221, 221, 227, 226, 227, 227, 226]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.289966192Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"":"downstairs","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"[2]R generation[0..12]: [144, 140, 140, 141, 144, 142, 142, 147, 146, 147, 147, 146]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.2899799Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"":"downstairs","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.289993978Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"":"downstairs","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.290018544Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"":"downstairs","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.290032092Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"":"downstairs","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"Next flush: 228","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.29004523Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"":"downstairs","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.290058388Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"":"downstairs","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.290071716Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.290083925Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"client":"0","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.290097203Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"client":"1","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.290110741Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"client":"2","":"downstairs","session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.290124059Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.290136517Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"6e503189-be8d-432e-aa5f-c5e15c694320 is now active with session: 9a830812-038c-4884-8d52-c4f7d006c787","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.290149465Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.290161843Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779,"session_id":"9a830812-038c-4884-8d52-c4f7d006c787"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:09.290174341Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 112 len 40960 data: 9 10 11 11 15 18 19 20 17 15 02/30 Read block 326 len 8192 03/30 Read block 293 len 36864 04/30 Write block 417 len 16384 data: 17 15 19 15 05/30 Write block 2 len 12288 data: 4 4 2 06/30 Flush 07/30 Write block 165 len 32768 data: 18 16 18 10 9 7 8 7 08/30 Flush 09/30 Write block 239 len 4096 data: 14 10/30 Read block 373 len 16384 11/30 Write block 236 len 36864 data: 14 15 14 15 15 15 14 16 18 12/30 Read block 501 len 32768 13/30 Read block 484 len 40960 14/30 Read block 521 len 28672 15/30 Write block 412 len 12288 data: 18 18 20 16/30 Read block 354 len 12288 17/30 Read block 230 len 32768 18/30 Flush 19/30 Write block 414 len 36864 data: 21 17 16 18 16 20 16 17 14 20/30 Write block 369 len 36864 data: 11 14 15 15 13 12 13 15 16 21/30 Read block 416 len 4096 22/30 Write block 436 len 12288 data: 11 7 10 23/30 Write block 79 len 40960 data: 12 13 13 12 12 11 19 16 13 14 24/30 Write block 122 len 16384 data: 15 18 15 13 25/30 Read block 482 len 20480 26/30 Write block 313 len 24576 data: 11 12 11 11 14 17 27/30 Write block 248 len 24576 data: 16 14 15 15 17 18 28/30 Read block 401 len 8192 29/30 Write block 306 len 36864 data: 9 9 9 9 10 12 12 12 13 30/30 Read block 454 len 20480 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 9 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 9 Done Done Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 12 11 10 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:36:09.309901651Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1779} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 228 228 228 F F F 1 020-039 140 140 140 216 216 216 F F F 2 040-059 140 140 140 216 216 216 F F F 3 060-079 141 141 141 217 217 217 T T T 4 080-099 144 144 144 224 224 224 T T T 5 100-119 148 148 148 228 228 228 F F F 6 120-139 148 148 148 228 228 228 T T T 7 140-159 147 147 147 227 227 227 F F F 8 160-179 148 148 148 229 229 229 F F F 9 180-199 147 147 147 227 227 227 F F F 10 200-219 147 147 147 227 227 227 F F F 11 220-239 148 148 148 230 230 230 F F F 12 240-259 148 148 148 230 230 230 T T T 13 260-279 147 147 147 227 227 227 F F F 14 280-299 137 137 137 210 210 210 F F F 15 300-319 142 142 142 220 220 220 T T T 16 320-339 144 144 144 223 223 223 F F F 17 340-359 144 144 144 223 223 223 F F F 18 360-379 147 147 147 227 227 227 T T T 19 380-399 145 145 145 225 225 225 F F F 20 400-419 148 148 148 230 230 230 T T T 21 420-439 148 148 148 228 228 228 T T T 22 440-459 147 147 147 227 227 227 F F F 23 460-479 146 146 146 226 226 226 F F F 24 480-499 142 142 142 220 220 220 F F F 25 500-519 144 144 144 224 224 224 F F F 26 520-539 144 144 144 224 224 224 F F F 27 540-559 147 147 147 227 227 227 F F F 28 560-579 143 143 143 222 222 222 F F F 29 580-599 134 134 134 205 205 205 F F F Max gen: 148, Max flush: 230 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 228 228 228 F F F 1 020-039 140 140 140 216 216 216 F F F 2 040-059 140 140 140 216 216 216 F F F 3 060-079 149 149 149 231 231 231 F F F 4 080-099 149 149 149 231 231 231 F F F 5 100-119 148 148 148 228 228 228 F F F 6 120-139 149 149 149 231 231 231 F F F 7 140-159 147 147 147 227 227 227 F F F 8 160-179 148 148 148 229 229 229 F F F 9 180-199 147 147 147 227 227 227 F F F 10 200-219 147 147 147 227 227 227 F F F 11 220-239 148 148 148 230 230 230 F F F 12 240-259 149 149 149 231 231 231 F F F 13 260-279 147 147 147 227 227 227 F F F 14 280-299 137 137 137 210 210 210 F F F 15 300-319 149 149 149 231 231 231 F F F 16 320-339 144 144 144 223 223 223 F F F 17 340-359 144 144 144 223 223 223 F F F 18 360-379 149 149 149 231 231 231 F F F 19 380-399 145 145 145 225 225 225 F F F 20 400-419 149 149 149 231 231 231 F F F 21 420-439 149 149 149 231 231 231 F F F 22 440-459 147 147 147 227 227 227 F F F 23 460-479 146 146 146 226 226 226 F F F 24 480-499 142 142 142 220 220 220 F F F 25 500-519 144 144 144 224 224 224 F F F 26 520-539 144 144 144 224 224 224 F F F 27 540-559 147 147 147 227 227 227 F F F 28 560-579 143 143 143 222 222 222 F F F 29 580-599 134 134 134 205 205 205 F F F Max gen: 149, Max flush: 231 Begin loop 75 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.627398936Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.67112935Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.671781395Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.67181873Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.671834007Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.671848425Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787} {"msg":"Crucible 834e5991-3e86-4a11-a1b0-089f0db24e29 has session id: 1869264a-1226-4c36-81f1-45422c809b6e","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.672242748Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"Upstairs opts: Upstairs UUID: 834e5991-3e86-4a11-a1b0-089f0db24e29, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.672291041Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"Crucible stats registered with UUID: 834e5991-3e86-4a11-a1b0-089f0db24e29","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.672313757Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.703027593Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.703058598Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787} {"msg":"834e5991-3e86-4a11-a1b0-089f0db24e29 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.703096013Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.70311314Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"client":"0","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.703141486Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"client":"0","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.703159494Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"client":"1","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.703177621Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"client":"1","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.703195258Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"client":"2","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.703212636Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"client":"2","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.703229593Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"":"io task","client":"2","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.703259879Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"":"io task","client":"0","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.703292804Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"ds_connection connected from Ok(127.0.0.1:46467)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.703330058Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"":"io task","client":"2","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"ds_connection connected from Ok(127.0.0.1:45433)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.703366813Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"ds_connection connected from Ok(127.0.0.1:48396)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.703545657Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"":"io task","client":"0","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.704201301Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"client":"2","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.704228407Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"client":"2","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.704381365Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"client":"1","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.704421679Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"client":"1","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.704452685Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"client":"0","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.70448081Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"client":"0","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.704984277Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"client":"2","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.705015152Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"client":"0","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.705164311Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"client":"1","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.705573991Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"client":"0","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.705603346Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"client":"0","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.705626723Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.70564912Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"client":"2","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.705674506Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"client":"2","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.705692144Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.705705062Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"client":"1","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.7057186Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"client":"1","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"[0]R flush_numbers[0..12]: [228, 216, 216, 231, 231, 228, 231, 227, 229, 227, 227, 230]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.705732478Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"":"downstairs","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"[0]R generation[0..12]: [148, 140, 140, 149, 149, 148, 149, 147, 148, 147, 147, 148]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.705746596Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"":"downstairs","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.705762054Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"":"downstairs","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"[1]R flush_numbers[0..12]: [228, 216, 216, 231, 231, 228, 231, 227, 229, 227, 227, 230]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.705776231Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"":"downstairs","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"[1]R generation[0..12]: [148, 140, 140, 149, 149, 148, 149, 147, 148, 147, 147, 148]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.705789839Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"":"downstairs","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.705803757Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"":"downstairs","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"[2]R flush_numbers[0..12]: [228, 216, 216, 231, 231, 228, 231, 227, 229, 227, 227, 230]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.705817615Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"":"downstairs","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"[2]R generation[0..12]: [148, 140, 140, 149, 149, 148, 149, 147, 148, 147, 147, 148]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.705831423Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"":"downstairs","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.705845331Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"":"downstairs","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.705859669Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"":"downstairs","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.705873257Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"":"downstairs","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":Wait for a query_work_queue command to finish before sending IO "Next flush: 232","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.705898114Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"":"downstairs","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.70592336Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"":"downstairs","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.705944797Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.705967144Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"client":"0","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.70598941Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"client":"1","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.706009817Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"client":"2","":"downstairs","session_id":"1869264a-1226-4c36-81f1-45422c809b6e"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 } {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.706028795Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.706041383Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"834e5991-3e86-4a11-a1b0-089f0db24e29 is now active with session: 1869264a-1226-4c36-81f1-45422c809b6e","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.706054431Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.706066619Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787,"session_id":"1869264a-1226-4c36-81f1-45422c809b6e"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:13.706078907Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 286 len 40960 data: 8 8 9 10 7 6 8 9 9 8 02/30 Read block 474 len 36864 03/30 Read block 423 len 24576 04/30 Read block 492 len 8192 05/30 Read block 499 len 40960 06/30 Read block 322 len 12288 07/30 Write block 0 len 24576 data: 2 3 5 5 3 4 08/30 Read block 249 len 36864 09/30 Read block 425 len 8192 10/30 Write block 283 len 40960 data: 10 7 9 9 9 10 11 8 7 9 11/30 Read block 569 len 32768 12/30 Read block 69 len 12288 13/30 Read block 445 len 16384 14/30 Read block 43 len 20480 15/30 Read block 134 len 36864 16/30 Read block 450 len 28672 17/30 Write block 245 len 4096 data: 18 18/30 Read block 68 len 32768 19/30 Read block 557 len 20480 20/30 Read block 412 len 32768 21/30 Read block 275 len 4096 22/30 Write block 174 len 8192 data: 8 10 23/30 Read block 73 len 12288 24/30 Read block 104 len 4096 25/30 Read block 61 len 16384 26/30 Write block 155 len 40960 data: 8 10 9 11 12 14 16 18 17 19 27/30 Write block 457 len 12288 data: 16 15 14 28/30 Read block 43 len 36864 29/30 Read block 459 len 4096 30/30 Write block 57 len 32768 data: 13 14 13 13 10 12 15 17 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Read 6 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Read 10 Done Done Done false 1005 Acked Read 3 Done Done Done false 1006 Acked Write 6 Done Done Done false 1007 Acked Read 9 Done Done Done false 1008 Acked Read 2 Done Done Done false 1009 Acked Write 10 Done Done Done false 1010 Acked Read 8 Done Done Done false 1011 Acked Read 3 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:36:13.726515693Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1787} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 228 228 228 T T T 1 020-039 140 140 140 216 216 216 F F F 2 040-059 140 140 140 216 216 216 T T T 3 060-079 149 149 149 231 231 231 T T T 4 080-099 149 149 149 231 231 231 F F F 5 100-119 148 148 148 228 228 228 F F F 6 120-139 149 149 149 231 231 231 F F F 7 140-159 147 147 147 227 227 227 T T T 8 160-179 148 148 148 229 229 229 T T T 9 180-199 147 147 147 227 227 227 F F F 10 200-219 147 147 147 227 227 227 F F F 11 220-239 148 148 148 230 230 230 F F F 12 240-259 149 149 149 231 231 231 T T T 13 260-279 147 147 147 227 227 227 F F F 14 280-299 137 137 137 210 210 210 T T T 15 300-319 149 149 149 231 231 231 F F F 16 320-339 144 144 144 223 223 223 F F F 17 340-359 144 144 144 223 223 223 F F F 18 360-379 149 149 149 231 231 231 F F F 19 380-399 145 145 145 225 225 225 F F F 20 400-419 149 149 149 231 231 231 F F F 21 420-439 149 149 149 231 231 231 F F F 22 440-459 147 147 147 227 227 227 T T T 23 460-479 146 146 146 226 226 226 F F F 24 480-499 142 142 142 220 220 220 F F F 25 500-519 144 144 144 224 224 224 F F F 26 520-539 144 144 144 224 224 224 F F F 27 540-559 147 147 147 227 227 227 F F F 28 560-579 143 143 143 222 222 222 F F F 29 580-599 134 134 134 205 205 205 F F F Max gen: 149, Max flush: 231 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 232 232 232 F F F 1 020-039 140 140 140 216 216 216 F F F 2 040-059 150 150 150 232 232 232 F F F 3 060-079 150 150 150 232 232 232 F F F 4 080-099 149 149 149 231 231 231 F F F 5 100-119 148 148 148 228 228 228 F F F 6 120-139 149 149 149 231 231 231 F F F 7 140-159 150 150 150 232 232 232 F F F 8 160-179 150 150 150 232 232 232 F F F 9 180-199 147 147 147 227 227 227 F F F 10 200-219 147 147 147 227 227 227 F F F 11 220-239 148 148 148 230 230 230 F F F 12 240-259 150 150 150 232 232 232 F F F 13 260-279 147 147 147 227 227 227 F F F 14 280-299 150 150 150 232 232 232 F F F 15 300-319 149 149 149 231 231 231 F F F 16 320-339 144 144 144 223 223 223 F F F 17 340-359 144 144 144 223 223 223 F F F 18 360-379 149 149 149 231 231 231 F F F 19 380-399 145 145 145 225 225 225 F F F 20 400-419 149 149 149 231 231 231 F F F 21 420-439 149 149 149 231 231 231 F F F 22 440-459 150 150 150 232 232 232 F F F 23 460-479 146 146 146 226 226 226 F F F 24 480-499 142 142 142 220 220 220 F F F 25 500-519 144 144 144 224 224 224 F F F 26 520-539 144 144 144 224 224 224 F F F 27 540-559 147 147 147 227 227 227 F F F 28 560-579 143 143 143 222 222 222 F F F 29 580-599 134 134 134 205 205 205 F F F Max gen: 150, Max flush: 232 Begin loop 76 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.012985037Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:36:18.055894972Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.056276176Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.088451148Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.08912288Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.089169103Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.08919341Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.089213647Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795} {"msg":"Crucible ef35009b-a149-49ca-b1b7-595d3e58fac6 has session id: 76931305-dd09-46c3-9390-1f851b200a33","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.089722353Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"Upstairs opts: Upstairs UUID: ef35009b-a149-49ca-b1b7-595d3e58fac6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.089768446Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"Crucible stats registered with UUID: ef35009b-a149-49ca-b1b7-595d3e58fac6","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.089796152Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.119899356Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.119932562Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795} {"msg":"ef35009b-a149-49ca-b1b7-595d3e58fac6 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.119966897Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.119985294Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"client":"0","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.12001459Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"client":"0","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.120032577Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"client":"1","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.120049704Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"client":"1","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.120066462Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"client":"2","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.120083479Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"client":"2","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.120100067Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"":"io task","client":"2","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.120121604Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"":"io task","client":"0","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.120159718Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"":"io task","client":"1","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"ds_connection connected from Ok(127.0.0.1:49434)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.12028387Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"":"io task","client":"2","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"ds_connection connected from Ok(127.0.0.1:46291)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.120330733Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"":"io task","client":"1","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"ds_connection connected from Ok(127.0.0.1:56357)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.120476622Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"":"io task","client":"0","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.121168381Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"client":"1","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.121196017Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"client":"1","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.121212464Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"client":"2","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.121232112Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"client":"2","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.121257598Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"client":"0","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.121288433Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"client":"0","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.121805008Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"client":"1","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.121963695Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"client":"0","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.122084347Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"client":"2","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.122446694Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"client":"1","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.122482449Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"client":"1","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.122514654Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.122530412Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"client":"0","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.12254445Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"client":"0","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.122557988Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.122597652Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"client":"2","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.12261557Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"client":"2","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"[0]R flush_numbers[0..12]: [232, 216, 232, 232, 231, 228, 231, 232, 232, 227, 227, 230]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.122630737Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"":"downstairs","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"[0]R generation[0..12]: [150, 140, 150, 150, 149, 148, 149, 150, 150, 147, 147, 148]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.122645115Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"":"downstairs","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.122659443Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"":"downstairs","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"[1]R flush_numbers[0..12]: [232, 216, 232, 232, 231, 228, 231, 232, 232, 227, 227, 230]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.122673471Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"":"downstairs","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"[1]R generation[0..12]: [150, 140, 150, 150, 149, 148, 149, 150, 150, 147, 147, 148]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.122687349Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"":"downstairs","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.122701387Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"":"downstairs","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"[2]R flush_numbers[0..12]: [232, 216, 232, 232, 231, 228, 231, 232, 232, 227, 227, 230]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.122715405Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"":"downstairs","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"[2]R generation[0..12]: [150, 140, 150, 150, 149, 148, 149, 150, 150, 147, 147, 148]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.122729263Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"":"downstairs","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.122745891Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"":"downstairs","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"Max found gen is 151","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.122759379Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"":"downstairs","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"Generation requested: 152 >= found:151","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.122772857Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"":"downstairs","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"Next flush: 233","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.122786075Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"":"downstairs","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:36:18.122799213Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"":"downstairs","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.1228211Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.122833548Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"client":"0","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.122847046Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"client":"1","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.122860344Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"client":"2","":"downstairs","session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.122873862Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.12288643Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"ef35009b-a149-49ca-b1b7-595d3e58fac6 is now active with session: 76931305-dd09-46c3-9390-1f851b200a33","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.122899538Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.122911747Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795,"session_id":"76931305-dd09-46c3-9390-1f851b200a33"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:18.122924345Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 483 len 28672 data: 11 11 11 9 11 8 10 02/30 Read block 498 len 24576 03/30 Write block 153 len 24576 data: 6 7 9 11 10 12 04/30 Write block 552 len 36864 data: 17 15 13 12 9 8 9 7 6 05/30 Read block 546 len 8192 06/30 Flush 07/30 Write block 361 len 32768 data: 11 9 9 12 15 13 13 12 08/30 Read block 32 len 4096 09/30 Flush 10/30 Write block 432 len 12288 data: 7 5 6 11/30 Write block 575 len 20480 data: 5 6 8 11 12 12/30 Read block 320 len 36864 13/30 Read block 591 len 12288 14/30 Write block 222 len 40960 data: 12 11 12 9 11 13 16 14 15 12 15/30 Read block 419 len 4096 16/30 Write block 453 len 36864 data: 11 13 14 12 17 16 15 15 15 17/30 Read block 292 len 28672 18/30 Write block 283 len 20480 data: 11 8 10 10 10 19/30 Write block 208 len 32768 data: 7 7 6 7 7 10 12 10 20/30 Read block 311 len 20480 21/30 Write block 90 len 16384 data: 16 14 14 12 22/30 Flush 23/30 Write block 349 len 32768 data: 13 14 14 13 12 11 8 8 24/30 Read block 471 len 16384 25/30 Write block 218 len 12288 data: 11 12 11 26/30 Read block 328 len 4096 27/30 Read block 415 len 24576 28/30 Write block 374 len 36864 data: 13 14 16 17 14 12 11 9 8 29/30 Write block 166 len 8192 data: 17 19 30/30 Write block 236 len 4096 data: 15 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 8 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 9 Sent Sent Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:36:18.143059854Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1795} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 232 232 232 F F F 1 020-039 140 140 140 216 216 216 F F F 2 040-059 150 150 150 232 232 232 F F F 3 060-079 150 150 150 232 232 232 F F F 4 080-099 152 152 152 235 235 235 F F F 5 100-119 148 148 148 228 228 228 F F F 6 120-139 149 149 149 231 231 231 F F F 7 140-159 152 152 152 233 233 233 F F F 8 160-179 150 150 150 232 232 232 T T T 9 180-199 147 147 147 227 227 227 F F F 10 200-219 152 152 152 235 235 235 T T T 11 220-239 152 152 152 235 235 235 T T T 12 240-259 150 150 150 232 232 232 F F F 13 260-279 147 147 147 227 227 227 F F F 14 280-299 152 152 152 235 235 235 F F F 15 300-319 149 149 149 231 231 231 F F F 16 320-339 144 144 144 223 223 223 F F F 17 340-359 144 144 144 223 223 223 T T T 18 360-379 152 152 152 234 234 234 T T T 19 380-399 145 145 145 225 225 225 T T T 20 400-419 149 149 149 231 231 231 F F F 21 420-439 152 152 152 235 235 235 F F F 22 440-459 152 152 152 235 235 235 F F F 23 460-479 152 152 152 235 235 235 F F F 24 480-499 152 152 152 233 233 233 F F F 25 500-519 144 144 144 224 224 224 F F F 26 520-539 144 144 144 224 224 224 F F F 27 540-559 152 152 152 233 233 233 F F F 28 560-579 152 152 152 235 235 235 F F F 29 580-599 134 134 134 205 205 205 F F F Max gen: 152, Max flush: 235 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 232 232 232 F F F 1 020-039 140 140 140 216 216 216 F F F 2 040-059 150 150 150 232 232 232 F F F 3 060-079 150 150 150 232 232 232 F F F 4 080-099 152 152 152 235 235 235 F F F 5 100-119 148 148 148 228 228 228 F F F 6 120-139 149 149 149 231 231 231 F F F 7 140-159 152 152 152 233 233 233 F F F 8 160-179 153 153 153 236 236 236 F F F 9 180-199 147 147 147 227 227 227 F F F 10 200-219 153 153 153 236 236 236 F F F 11 220-239 153 153 153 236 236 236 F F F 12 240-259 150 150 150 232 232 232 F F F 13 260-279 147 147 147 227 227 227 F F F 14 280-299 152 152 152 235 235 235 F F F 15 300-319 149 149 149 231 231 231 F F F 16 320-339 144 144 144 223 223 223 F F F 17 340-359 153 153 153 236 236 236 F F F 18 360-379 153 153 153 236 236 236 F F F 19 380-399 153 153 153 236 236 236 F F F 20 400-419 149 149 149 231 231 231 F F F 21 420-439 152 152 152 235 235 235 F F F 22 440-459 152 152 152 235 235 235 F F F 23 460-479 152 152 152 235 235 235 F F F 24 480-499 152 152 152 233 233 233 F F F 25 500-519 144 144 144 224 224 224 F F F 26 520-539 144 144 144 224 224 224 F F F 27 540-559 152 152 152 233 233 233 F F F 28 560-579 152 152 152 235 235 235 F F F 29 580-599 134 134 134 205 205 205 F F F Max gen: 153, Max flush: 236 Begin loop 77 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.421185677Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.464729219Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.465393462Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.465431216Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.465446194Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.465461462Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803} {"msg":"Crucible 36e13935-91ab-48dc-92d6-ff2bca39a6bd has session id: ab102709-88ba-4e40-84f3-5f7f1b454b63","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.465950561Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"Upstairs opts: Upstairs UUID: 36e13935-91ab-48dc-92d6-ff2bca39a6bd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.465991244Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"Crucible stats registered with UUID: 36e13935-91ab-48dc-92d6-ff2bca39a6bd","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.466007912Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.49752886Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.497567214Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803} {"msg":"36e13935-91ab-48dc-92d6-ff2bca39a6bd active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.497604419Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.497621966Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"client":"0","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.497650712Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"client":"0","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.497668639Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"client":"1","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.497685587Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"client":"1","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.497702524Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"client":"2","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.497722221Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"client":"2","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.497754307Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"":"io task","client":"2","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.497789752Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"":"io task","client":"0","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.497824696Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"":"io task","client":"1","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"ds_connection connected from Ok(127.0.0.1:36748)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.497861371Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"":"io task","client":"2","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"ds_connection connected from Ok(127.0.0.1:54486)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.497899536Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"":"io task","client":"0","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"ds_connection connected from Ok(127.0.0.1:59504)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.49793625Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"":"io task","client":"1","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.49862392Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"client":"2","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.498664614Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"client":"2","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.498794905Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"client":"0","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.498867884Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"client":"0","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.498971119Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"client":"1","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.499005924Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"client":"1","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.499300831Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"client":"2","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.499543985Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"client":"0","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.499676456Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"client":"1","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.499817565Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"client":"2","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.49985617Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"client":"2","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.499881896Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.500053961Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"client":"0","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.500084057Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"client":"0","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.500102534Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.500211848Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"client":"1","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.500232355Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"client":"1","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"[0]R flush_numbers[0..12]: [232, 216, 232, 232, 235, 228, 231, 233, 236, 227, 236, 236]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.500247153Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"":"downstairs","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"[0]R generation[0..12]: [150, 140, 150, 150, 152, 148, 149, 152, 153, 147, 153, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.500261631Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"":"downstairs","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.500277478Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"":"downstairs","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"[1]R flush_numbers[0..12]: [232, 216, 232, 232, 235, 228, 231, 233, 236, 227, 236, 236]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.500291496Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"":"downstairs","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"[1]R generation[0..12]: [150, 140, 150, 150, 152, 148, 149, 152, 153, 147, 153, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.500305244Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"":"downstairs","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.500319412Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"":"downstairs","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"[2]R flush_numbers[0..12]: [232, 216, 232, 232, 235, 228, 231, 233, 236, 227, 236, 236]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.50033333Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"":"downstairs","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"[2]R generation[0..12]: [150, 140, 150, 150, 152, 148, 149, 152, 153, 147, 153, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.500347308Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"":"downstairs","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.500362646Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"":"downstairs","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.500376124Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"":"downstairs","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.500389462Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"":"downstairs","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"Next flush: 237","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.50040518Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"":"downstairs","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-01-03T08:36:22.500418528Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"":"downstairs","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.500439925Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.500452083Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"client":"0","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.500465411Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"client":"1","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.500478539Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"client":"2","":"downstairs","session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.500491857Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.500504235Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"36e13935-91ab-48dc-92d6-ff2bca39a6bd is now active with session: ab102709-88ba-4e40-84f3-5f7f1b454b63","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.500517153Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.500529381Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803,"session_id":"ab102709-88ba-4e40-84f3-5f7f1b454b63"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:22.50054179Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 245 len 24576 data: 19 16 17 17 15 16 02/30 Write block 317 len 12288 data: 15 18 14 03/30 Read block 481 len 28672 04/30 Read block 244 len 28672 05/30 Write block 185 len 32768 data: 17 20 17 17 15 16 15 11 06/30 Write block 17 len 12288 data: 13 13 14 07/30 Write block 329 len 8192 data: 10 10 08/30 Write block 371 len 20480 data: 16 16 14 14 15 09/30 Read block 253 len 8192 10/30 Read block 178 len 32768 11/30 Read block 457 len 4096 12/30 Read block 261 len 24576 13/30 Read block 248 len 8192 14/30 Read block 18 len 28672 15/30 Read block 41 len 32768 16/30 Read block 44 len 24576 17/30 Read block 83 len 4096 18/30 Write block 263 len 20480 data: 16 15 13 16 15 19/30 Write block 455 len 20480 data: 15 13 18 17 16 20/30 Write block 486 len 16384 data: 10 12 9 11 21/30 Read block 348 len 8192 22/30 Flush 23/30 Write block 522 len 12288 data: 12 13 14 24/30 Read block 428 len 32768 25/30 Read block 50 len 32768 26/30 Read block 161 len 24576 27/30 Read block 131 len 16384 28/30 Write block 34 len 24576 data: 8 4 5 8 8 12 29/30 Write block 261 len 12288 data: 16 16 17 30/30 Read block 458 len 20480 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 3 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 3 Sent Done Done false 1029 Acked Read 5 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 0 2 Done 6 8 8 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:36:22.519741386Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1803} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 237 237 237 F F F 1 020-039 140 140 140 216 216 216 T T T 2 040-059 150 150 150 232 232 232 F F F 3 060-079 150 150 150 232 232 232 F F F 4 080-099 152 152 152 235 235 235 F F F 5 100-119 148 148 148 228 228 228 F F F 6 120-139 149 149 149 231 231 231 F F F 7 140-159 152 152 152 233 233 233 F F F 8 160-179 153 153 153 236 236 236 F F F 9 180-199 154 154 154 237 237 237 F F F 10 200-219 153 153 153 236 236 236 F F F 11 220-239 153 153 153 236 236 236 F F F 12 240-259 154 154 154 237 237 237 F F F 13 260-279 154 154 154 237 237 237 T T T 14 280-299 152 152 152 235 235 235 F F F 15 300-319 154 154 154 237 237 237 F F F 16 320-339 154 154 154 237 237 237 F F F 17 340-359 153 153 153 236 236 236 F F F 18 360-379 154 154 154 237 237 237 F F F 19 380-399 153 153 153 236 236 236 F F F 20 400-419 149 149 149 231 231 231 F F F 21 420-439 152 152 152 235 235 235 F F F 22 440-459 154 154 154 237 237 237 F F F 23 460-479 152 152 152 235 235 235 F F F 24 480-499 154 154 154 237 237 237 F F F 25 500-519 144 144 144 224 224 224 F F F 26 520-539 144 144 144 224 224 224 T T T 27 540-559 152 152 152 233 233 233 F F F 28 560-579 152 152 152 235 235 235 F F F 29 580-599 134 134 134 205 205 205 F F F Max gen: 154, Max flush: 237 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 237 237 237 F F F 1 020-039 155 155 155 238 238 238 F F F 2 040-059 150 150 150 232 232 232 F F F 3 060-079 150 150 150 232 232 232 F F F 4 080-099 152 152 152 235 235 235 F F F 5 100-119 148 148 148 228 228 228 F F F 6 120-139 149 149 149 231 231 231 F F F 7 140-159 152 152 152 233 233 233 F F F 8 160-179 153 153 153 236 236 236 F F F 9 180-199 154 154 154 237 237 237 F F F 10 200-219 153 153 153 236 236 236 F F F 11 220-239 153 153 153 236 236 236 F F F 12 240-259 154 154 154 237 237 237 F F F 13 260-279 155 155 155 238 238 238 F F F 14 280-299 152 152 152 235 235 235 F F F 15 300-319 154 154 154 237 237 237 F F F 16 320-339 154 154 154 237 237 237 F F F 17 340-359 153 153 153 236 236 236 F F F 18 360-379 154 154 154 237 237 237 F F F 19 380-399 153 153 153 236 236 236 F F F 20 400-419 149 149 149 231 231 231 F F F 21 420-439 152 152 152 235 235 235 F F F 22 440-459 154 154 154 237 237 237 F F F 23 460-479 152 152 152 235 235 235 F F F 24 480-499 154 154 154 237 237 237 F F F 25 500-519 144 144 144 224 224 224 F F F 26 520-539 155 155 155 238 238 238 F F F 27 540-559 152 152 152 233 233 233 F F F 28 560-579 152 152 152 235 235 235 F F F 29 580-599 134 134 134 205 205 205 F F F Max gen: 155, Max flush: 238 Begin loop 78 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.778011488Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:36:26.82066825Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.821048355Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.852814906Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.853381583Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.853426767Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.853444824Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.853462962Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811} {"msg":"Crucible c1a62a01-7b95-4698-aa8a-34b74f7bf8ed has session id: d2bba405-f319-46ab-ac74-9936b4457bc1","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.853878431Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"Upstairs opts: Upstairs UUID: c1a62a01-7b95-4698-aa8a-34b74f7bf8ed, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.853933563Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"Crucible stats registered with UUID: c1a62a01-7b95-4698-aa8a-34b74f7bf8ed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.853970917Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.884398625Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.88442783Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811} {"msg":"c1a62a01-7b95-4698-aa8a-34b74f7bf8ed active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.884453497Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.884469784Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"client":"0","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.88449774Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"client":"0","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.884516237Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"client":"1","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.884533795Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"client":"1","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.884555431Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"client":"2","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.884588117Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"client":"2","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.884619862Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"":"io task","client":"2","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.884647038Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"":"io task","client":"0","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.884679633Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"":"io task","client":"1","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"ds_connection connected from Ok(127.0.0.1:45921)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.88483946Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"":"io task","client":"1","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"ds_connection connected from Ok(127.0.0.1:49580)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.884882494Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"":"io task","client":"2","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"ds_connection connected from Ok(127.0.0.1:38330)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.884915849Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"":"io task","client":"0","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.885541767Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"client":"2","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.885570853Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"client":"2","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.885587031Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"client":"0","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.885601289Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"client":"0","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.885614907Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"client":"1","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.885632504Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"client":"1","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.886292008Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"client":"1","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.886322984Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"client":"2","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.886339211Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"client":"0","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.886884962Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"client":"1","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.886913097Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"client":"1","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.886927765Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.886940443Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"client":"0","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.886954391Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"client":"0","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.886968309Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.886984077Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"client":"2","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.886998175Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"client":"2","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"[0]R flush_numbers[0..12]: [237, 238, 232, 232, 235, 228, 231, 233, 236, 237, 236, 236]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.887012463Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"":"downstairs","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"[0]R generation[0..12]: [154, 155, 150, 150, 152, 148, 149, 152, 153, 154, 153, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.887026441Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"":"downstairs","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.887040669Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"":"downstairs","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"[1]R flush_numbers[0..12]: [237, 238, 232, 232, 235, 228, 231, 233, 236, 237, 236, 236]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.887054457Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"":"downstairs","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"[1]R generation[0..12]: [154, 155, 150, 150, 152, 148, 149, 152, 153, 154, 153, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.887068195Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"":"downstairs","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.887082173Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"":"downstairs","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"[2]R flush_numbers[0..12]: [237, 238, 232, 232, 235, 228, 231, 233, 236, 237, 236, 236]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.887096101Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"":"downstairs","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"[2]R generation[0..12]: [154, 155, 150, 150, 152, 148, 149, 152, 153, 154, 153, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.887110098Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"":"downstairs","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.887124107Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"":"downstairs","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.887137055Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"":"downstairs","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2025-01-03T08:36:26.887150313Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"":"downstairs","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"Next flush: 239","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.887174009Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"":"downstairs","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.887189007Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"":"downstairs","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.887202465Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.887214523Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"client":"0","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.887227881Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"client":"1","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.88724099Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"client":"2","":"downstairs","session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.887254548Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.887266796Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"c1a62a01-7b95-4698-aa8a-34b74f7bf8ed is now active with session: d2bba405-f319-46ab-ac74-9936b4457bc1","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.887279824Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.887291952Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811,"session_id":"d2bba405-f319-46ab-ac74-9936b4457bc1"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:26.88730443Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 411 len 40960 data: 18 19 19 22 18 17 19 17 21 17 02/30 Read block 324 len 36864 03/30 Read block 452 len 8192 04/30 Read block 464 len 24576 05/30 Flush 06/30 Write block 441 len 16384 data: 12 13 13 13 07/30 Read block 485 len 32768 08/30 Read block 290 len 12288 09/30 Read block 433 len 20480 10/30 Read block 442 len 40960 11/30 Write block 156 len 8192 data: 12 11 12/30 Read block 535 len 4096 13/30 Read block 71 len 4096 14/30 Write block 65 len 20480 data: 15 15 15 15 12 15/30 Read block 364 len 32768 16/30 Write block 210 len 32768 data: 7 8 8 11 13 11 11 11 17/30 Write block 426 len 12288 data: 8 8 8 18/30 Read block 367 len 36864 19/30 Write block 220 len 24576 data: 12 13 13 12 13 10 20/30 Read block 491 len 16384 21/30 Read block 228 len 20480 22/30 Read block 333 len 16384 23/30 Write block 505 len 28672 data: 9 9 10 12 10 10 10 24/30 Write block 273 len 4096 data: 12 25/30 Read block 579 len 36864 26/30 Write block 372 len 16384 data: 17 15 15 16 27/30 Write block 573 len 20480 data: 11 5 6 7 9 28/30 Write block 466 len 32768 data: 10 11 9 11 12 14 14 15 29/30 Write block 335 len 16384 data: 7 7 7 8 30/30 Read block 252 len 28672 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 4 Done Done Done false 1006 Acked Read 8 Done Done Done false 1007 Acked Read 3 Done Done Done false 1008 Acked Read 5 Done Done Done false 1009 Acked Read 10 Done Done Done false 1010 Acked Write 2 Done Done Done false 1011 Acked Read 1 Done Done Done false 1012 Acked Read 1 Done Done Done false 1013 Acked Write 5 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 4 Done Sent Done false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 1 3 Done 25 23 24 72 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1004 1004 1004 Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:36:26.906724674Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1811} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 237 237 237 F F F 1 020-039 155 155 155 238 238 238 F F F 2 040-059 150 150 150 232 232 232 F F F 3 060-079 150 150 150 232 232 232 T T T 4 080-099 152 152 152 235 235 235 F F F 5 100-119 148 148 148 228 228 228 F F F 6 120-139 149 149 149 231 231 231 F F F 7 140-159 152 152 152 233 233 233 T T T 8 160-179 153 153 153 236 236 236 F F F 9 180-199 154 154 154 237 237 237 F F F 10 200-219 153 153 153 236 236 236 T T T 11 220-239 153 153 153 236 236 236 T T T 12 240-259 154 154 154 237 237 237 F F F 13 260-279 155 155 155 238 238 238 T T T 14 280-299 152 152 152 235 235 235 F F F 15 300-319 154 154 154 237 237 237 F F F 16 320-339 154 154 154 237 237 237 T T T 17 340-359 153 153 153 236 236 236 F F F 18 360-379 154 154 154 237 237 237 T T T 19 380-399 153 153 153 236 236 236 F F F 20 400-419 156 156 156 239 239 239 F F F 21 420-439 156 156 156 239 239 239 T T T 22 440-459 154 154 154 237 237 237 T T T 23 460-479 152 152 152 235 235 235 T T T 24 480-499 154 154 154 237 237 237 F F F 25 500-519 144 144 144 224 224 224 T T T 26 520-539 155 155 155 238 238 238 F F F 27 540-559 152 152 152 233 233 233 F F F 28 560-579 152 152 152 235 235 235 T T T 29 580-599 134 134 134 205 205 205 F F F Max gen: 156, Max flush: 239 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 237 237 237 F F F 1 020-039 155 155 155 238 238 238 F F F 2 040-059 150 150 150 232 232 232 F F F 3 060-079 157 157 157 240 240 240 F F F 4 080-099 152 152 152 235 235 235 F F F 5 100-119 148 148 148 228 228 228 F F F 6 120-139 149 149 149 231 231 231 F F F 7 140-159 157 157 157 240 240 240 F F F 8 160-179 153 153 153 236 236 236 F F F 9 180-199 154 154 154 237 237 237 F F F 10 200-219 157 157 157 240 240 240 F F F 11 220-239 157 157 157 240 240 240 F F F 12 240-259 154 154 154 237 237 237 F F F 13 260-279 157 157 157 240 240 240 F F F 14 280-299 152 152 152 235 235 235 F F F 15 300-319 154 154 154 237 237 237 F F F 16 320-339 157 157 157 240 240 240 F F F 17 340-359 153 153 153 236 236 236 F F F 18 360-379 157 157 157 240 240 240 F F F 19 380-399 153 153 153 236 236 236 F F F 20 400-419 156 156 156 239 239 239 F F F 21 420-439 157 157 157 240 240 240 F F F 22 440-459 157 157 157 240 240 240 F F F 23 460-479 157 157 157 240 240 240 F F F 24 480-499 154 154 154 237 237 237 F F F 25 500-519 157 157 157 240 240 240 F F F 26 520-539 155 155 155 238 238 238 F F F 27 540-559 152 152 152 233 233 233 F F F 28 560-579 157 157 157 240 240 240 F F F 29 580-599 134 134 134 205 205 205 F F F Max gen: 157, Max flush: 240 Begin loop 79 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.24421707Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.288237663Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.288983163Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.289016719Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.289030747Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.289045494Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819} {"msg":"Crucible 40fb771c-b948-462d-a291-47482132ec0a has session id: feb31141-8412-45ba-a8d6-6ede30f70ab1","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.289510537Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"Upstairs opts: Upstairs UUID: 40fb771c-b948-462d-a291-47482132ec0a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.289549911Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"Crucible stats registered with UUID: 40fb771c-b948-462d-a291-47482132ec0a","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.289570618Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.320428932Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.320462047Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819} {"msg":"40fb771c-b948-462d-a291-47482132ec0a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.320492203Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.32050944Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"client":"0","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.320538496Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"client":"0","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.320556673Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"client":"1","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.320574031Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"client":"1","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.320590779Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"client":"2","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.320607686Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"client":"2","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.320624243Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"":"io task","client":"2","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.320641721Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"":"io task","client":"0","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.320667177Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"":"io task","client":"1","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"ds_connection connected from Ok(127.0.0.1:35085)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.320689464Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"":"io task","client":"2","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"ds_connection connected from Ok(127.0.0.1:57578)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.320731208Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"":"io task","client":"0","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"ds_connection connected from Ok(127.0.0.1:51815)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.320772972Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"":"io task","client":"1","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.321611929Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"client":"0","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.321647604Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"client":"0","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.32167699Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"client":"2","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.321706985Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"client":"2","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.321736581Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"client":"1","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.321763757Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"client":"1","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.322251666Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"client":"0","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.322282101Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"client":"2","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.32235733Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"client":"1","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.322813814Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"client":"0","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.32284278Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"client":"0","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.322857968Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.322992628Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"client":"1","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.323020134Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"client":"1","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.323034811Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.323051059Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"client":"2","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.323065467Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"client":"2","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"[0]R flush_numbers[0..12]: [237, 238, 232, 240, 235, 228, 231, 240, 236, 237, 240, 240]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.323079635Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"":"downstairs","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"[0]R generation[0..12]: [154, 155, 150, 157, 152, 148, 149, 157, 153, 154, 157, 157]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.323093603Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"":"downstairs","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.323107711Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"":"downstairs","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"[1]R flush_numbers[0..12]: [237, 238, 232, 240, 235, 228, 231, 240, 236, 237, 240, 240]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.323121749Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"":"downstairs","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"[1]R generation[0..12]: [154, 155, 150, 157, 152, 148, 149, 157, 153, 154, 157, 157]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.323135807Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"":"downstairs","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.323149785Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"":"downstairs","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"[2]R flush_numbers[0..12]: [237, 238, 232, 240, 235, 228, 231, 240, 236, 237, 240, 240]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.323163583Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"":"downstairs","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"[2]R generation[0..12]: [154, 155, 150, 157, 152, 148, 149, 157, 153, 154, 157, 157]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.323177401Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"":"downstairs","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.323191429Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"":"downstairs","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:36:31.323204567Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"":"downstairs","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.323224064Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"":"downstairs","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"Next flush: 241","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.323239102Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"":"downstairs","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.32325248Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"":"downstairs","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.323265918Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.323277936Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"client":"0","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.323291364Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"client":"1","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.323304512Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"client":"2","":"downstairs","session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.32331786Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.323330368Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"40fb771c-b948-462d-a291-47482132ec0a is now active with session: feb31141-8412-45ba-a8d6-6ede30f70ab1","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.323343377Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.323355505Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819,"session_id":"feb31141-8412-45ba-a8d6-6ede30f70ab1"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:31.323367803Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 67 len 20480 data: 16 16 13 14 12 02/30 Flush 03/30 Write block 577 len 12288 data: 10 12 13 04/30 Read block 545 len 32768 05/30 Read block 322 len 36864 06/30 Write block 481 len 20480 data: 11 11 12 12 12 07/30 Read block 485 len 28672 08/30 Write block 254 len 36864 data: 18 14 18 17 17 16 16 17 17 09/30 Read block 476 len 28672 10/30 Write block 254 len 4096 data: 19 11/30 Write block 4 len 20480 data: 4 5 4 4 7 12/30 Read block 183 len 4096 13/30 Read block 244 len 24576 14/30 Flush 15/30 Write block 468 len 24576 data: 10 12 13 15 15 16 16/30 Read block 66 len 28672 17/30 Read block 297 len 32768 18/30 Flush 19/30 Write block 314 len 40960 data: 14 12 12 16 19 15 12 11 11 12 20/30 Read block 99 len 28672 21/30 Read block 433 len 32768 22/30 Write block 212 len 16384 data: 9 12 14 12 23/30 Write block 380 len 4096 data: 12 24/30 Read block 387 len 20480 25/30 Flush 26/30 Write block 449 len 4096 data: 13 27/30 Read block 114 len 32768 28/30 Read block 125 len 24576 29/30 Write block 575 len 28672 data: 7 8 11 13 14 11 11 30/30 Read block 188 len 4096 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 1 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:36:31.343219015Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1819} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 242 242 242 F F F 1 020-039 155 155 155 238 238 238 F F F 2 040-059 150 150 150 232 232 232 F F F 3 060-079 158 158 158 241 241 241 F F F 4 080-099 152 152 152 235 235 235 F F F 5 100-119 148 148 148 228 228 228 F F F 6 120-139 149 149 149 231 231 231 F F F 7 140-159 157 157 157 240 240 240 F F F 8 160-179 153 153 153 236 236 236 F F F 9 180-199 154 154 154 237 237 237 F F F 10 200-219 158 158 158 244 244 244 F F F 11 220-239 157 157 157 240 240 240 F F F 12 240-259 158 158 158 242 242 242 F F F 13 260-279 158 158 158 242 242 242 F F F 14 280-299 152 152 152 235 235 235 F F F 15 300-319 158 158 158 244 244 244 F F F 16 320-339 158 158 158 244 244 244 F F F 17 340-359 153 153 153 236 236 236 F F F 18 360-379 157 157 157 240 240 240 F F F 19 380-399 158 158 158 244 244 244 F F F 20 400-419 156 156 156 239 239 239 F F F 21 420-439 157 157 157 240 240 240 F F F 22 440-459 157 157 157 240 240 240 T T T 23 460-479 158 158 158 243 243 243 F F F 24 480-499 158 158 158 242 242 242 F F F 25 500-519 157 157 157 240 240 240 F F F 26 520-539 155 155 155 238 238 238 F F F 27 540-559 152 152 152 233 233 233 F F F 28 560-579 158 158 158 242 242 242 T T T 29 580-599 134 134 134 205 205 205 T T T Max gen: 158, Max flush: 244 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 242 242 242 F F F 1 020-039 155 155 155 238 238 238 F F F 2 040-059 150 150 150 232 232 232 F F F 3 060-079 158 158 158 241 241 241 F F F 4 080-099 152 152 152 235 235 235 F F F 5 100-119 148 148 148 228 228 228 F F F 6 120-139 149 149 149 231 231 231 F F F 7 140-159 157 157 157 240 240 240 F F F 8 160-179 153 153 153 236 236 236 F F F 9 180-199 154 154 154 237 237 237 F F F 10 200-219 158 158 158 244 244 244 F F F 11 220-239 157 157 157 240 240 240 F F F 12 240-259 158 158 158 242 242 242 F F F 13 260-279 158 158 158 242 242 242 F F F 14 280-299 152 152 152 235 235 235 F F F 15 300-319 158 158 158 244 244 244 F F F 16 320-339 158 158 158 244 244 244 F F F 17 340-359 153 153 153 236 236 236 F F F 18 360-379 157 157 157 240 240 240 F F F 19 380-399 158 158 158 244 244 244 F F F 20 400-419 156 156 156 239 239 239 F F F 21 420-439 157 157 157 240 240 240 F F F 22 440-459 159 159 159 245 245 245 F F F 23 460-479 158 158 158 243 243 243 F F F 24 480-499 158 158 158 242 242 242 F F F 25 500-519 157 157 157 240 240 240 F F F 26 520-539 155 155 155 238 238 238 F F F 27 540-559 152 152 152 233 233 233 F F F 28 560-579 159 159 159 245 245 245 F F F 29 580-599 159 159 159 245 245 245 F F F Max gen: 159, Max flush: 245 Begin loop 80 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.599794645Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.643744738Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.644407541Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.644445755Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.644460983Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.644475351Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827} {"msg":"Crucible 5a565ee6-41c0-447f-bd02-6b917b181349 has session id: 65230002-5fcc-47d9-aa2f-5e0c51aba33c","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.644910768Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"Upstairs opts: Upstairs UUID: 5a565ee6-41c0-447f-bd02-6b917b181349, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.644956461Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"Crucible stats registered with UUID: 5a565ee6-41c0-447f-bd02-6b917b181349","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.644995785Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.675755054Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.675785719Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827} {"msg":"5a565ee6-41c0-447f-bd02-6b917b181349 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.675817885Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.675839112Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"client":"0","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.675867028Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"client":"0","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.675884935Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"client":"1","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.675902252Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"client":"1","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.67591949Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"client":"2","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.675936927Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"client":"2","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.675954185Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"":"io task","client":"2","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.675971972Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"":"io task","client":"0","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.675990039Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"":"io task","client":"1","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"ds_connection connected from Ok(127.0.0.1:39958)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.676043472Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"":"io task","client":"2","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"ds_connection connected from Ok(127.0.0.1:37472)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.676141877Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"":"io task","client":"1","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"ds_connection connected from Ok(127.0.0.1:52257)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.676305133Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"":"io task","client":"0","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.676748579Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"client":"0","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.676789143Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"client":"0","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.676820658Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"client":"1","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.676850134Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"client":"1","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.677159129Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"client":"2","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.677202622Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"client":"2","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.677406143Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"client":"1","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.677545292Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"client":"0","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.677874534Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"client":"2","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.678035921Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"client":"1","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.678057088Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"client":"1","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.678071686Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.6781805Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"client":"0","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.678199567Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"client":"0","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.678213235Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.678474747Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"client":"2","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.678499113Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"client":"2","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"[0]R flush_numbers[0..12]: [242, 238, 232, 241, 235, 228, 231, 240, 236, 237, 244, 240]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.678513701Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"":"downstairs","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"[0]R generation[0..12]: [158, 155, 150, 158, 152, 148, 149, 157, 153, 154, 158, 157]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.678527739Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"":"downstairs","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.678545396Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"":"downstairs","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"[1]R flush_numbers[0..12]: [242, 238, 232, 241, 235, 228, 231, 240, 236, 237, 244, 240]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.678559624Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"":"downstairs","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"[1]R generation[0..12]: [158, 155, 150, 158, 152, 148, 149, 157, 153, 154, 158, 157]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.678573772Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"":"downstairs","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.67858791Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"":"downstairs","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"[2]R flush_numbers[0..12]: [242, 238, 232, 241, 235, 228, 231, 240, 236, 237, 244, 240]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.678601918Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"":"downstairs","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"[2]R generation[0..12]: [158, 155, 150, 158, 152, 148, 149, 157, 153, 154, 158, 157]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.678615696Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"":"downstairs","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.678629614Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"":"downstairs","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.678642632Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"":"downstairs","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.67865571Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"":"downstairs","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"Next flush: 246","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:36:35.678668708Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"":"downstairs","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.678685556Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"":"downstairs","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.678699214Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.678711362Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"client":"0","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.67872495Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"client":"1","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.678740088Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"client":"2","":"downstairs","session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.678754026Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.678766504Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"5a565ee6-41c0-447f-bd02-6b917b181349 is now active with session: 65230002-5fcc-47d9-aa2f-5e0c51aba33c","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.678779412Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.678791641Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827,"session_id":"65230002-5fcc-47d9-aa2f-5e0c51aba33c"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:35.678804049Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 152 len 28672 data: 7 7 8 10 13 12 13 02/30 Write block 304 len 8192 data: 9 8 03/30 Flush 04/30 Flush 05/30 Flush 06/30 Write block 262 len 8192 data: 18 18 07/30 Read block 404 len 24576 08/30 Read block 393 len 32768 09/30 Read block 52 len 24576 10/30 Read block 339 len 16384 11/30 Flush 12/30 Write block 251 len 32768 data: 16 18 19 20 15 19 18 18 13/30 Flush 14/30 Write block 534 len 4096 data: 10 15/30 Write block 476 len 32768 data: 15 13 11 10 12 12 12 13 16/30 Flush 17/30 Write block 184 len 20480 data: 18 18 21 18 18 18/30 Read block 382 len 16384 19/30 Read block 336 len 4096 20/30 Write block 316 len 4096 data: 13 21/30 Write block 75 len 12288 data: 16 17 16 22/30 Write block 91 len 20480 data: 15 15 13 10 11 23/30 Read block 121 len 16384 24/30 Flush 25/30 Write block 536 len 40960 data: 7 7 6 6 7 9 12 13 14 15 26/30 Read block 468 len 20480 27/30 Write block 84 len 8192 data: 12 20 28/30 Write block 140 len 24576 data: 8 10 12 14 15 16 29/30 Read block 31 len 20480 30/30 Read block 357 len 28672 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 10 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 7 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 6 5 6 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:36:35.697701159Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1827} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 242 242 242 F F F 1 020-039 155 155 155 238 238 238 F F F 2 040-059 150 150 150 232 232 232 F F F 3 060-079 160 160 160 252 252 252 F F F 4 080-099 160 160 160 252 252 252 T T T 5 100-119 148 148 148 228 228 228 F F F 6 120-139 149 149 149 231 231 231 F F F 7 140-159 160 160 160 246 246 246 T T T 8 160-179 153 153 153 236 236 236 F F F 9 180-199 160 160 160 252 252 252 F F F 10 200-219 158 158 158 244 244 244 F F F 11 220-239 157 157 157 240 240 240 F F F 12 240-259 160 160 160 250 250 250 F F F 13 260-279 160 160 160 249 249 249 F F F 14 280-299 152 152 152 235 235 235 F F F 15 300-319 160 160 160 252 252 252 F F F 16 320-339 158 158 158 244 244 244 F F F 17 340-359 153 153 153 236 236 236 F F F 18 360-379 157 157 157 240 240 240 F F F 19 380-399 158 158 158 244 244 244 F F F 20 400-419 156 156 156 239 239 239 F F F 21 420-439 157 157 157 240 240 240 F F F 22 440-459 159 159 159 245 245 245 F F F 23 460-479 160 160 160 251 251 251 F F F 24 480-499 160 160 160 251 251 251 F F F 25 500-519 157 157 157 240 240 240 F F F 26 520-539 160 160 160 251 251 251 T T T 27 540-559 152 152 152 233 233 233 T T T 28 560-579 159 159 159 245 245 245 F F F 29 580-599 159 159 159 245 245 245 F F F Max gen: 160, Max flush: 252 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 242 242 242 F F F 1 020-039 155 155 155 238 238 238 F F F 2 040-059 150 150 150 232 232 232 F F F 3 060-079 160 160 160 252 252 252 F F F 4 080-099 161 161 161 253 253 253 F F F 5 100-119 148 148 148 228 228 228 F F F 6 120-139 149 149 149 231 231 231 F F F 7 140-159 161 161 161 253 253 253 F F F 8 160-179 153 153 153 236 236 236 F F F 9 180-199 160 160 160 252 252 252 F F F 10 200-219 158 158 158 244 244 244 F F F 11 220-239 157 157 157 240 240 240 F F F 12 240-259 160 160 160 250 250 250 F F F 13 260-279 160 160 160 249 249 249 F F F 14 280-299 152 152 152 235 235 235 F F F 15 300-319 160 160 160 252 252 252 F F F 16 320-339 158 158 158 244 244 244 F F F 17 340-359 153 153 153 236 236 236 F F F 18 360-379 157 157 157 240 240 240 F F F 19 380-399 158 158 158 244 244 244 F F F 20 400-419 156 156 156 239 239 239 F F F 21 420-439 157 157 157 240 240 240 F F F 22 440-459 159 159 159 245 245 245 F F F 23 460-479 160 160 160 251 251 251 F F F 24 480-499 160 160 160 251 251 251 F F F 25 500-519 157 157 157 240 240 240 F F F 26 520-539 161 161 161 253 253 253 F F F 27 540-559 161 161 161 253 253 253 F F F 28 560-579 159 159 159 245 245 245 F F F 29 580-599 159 159 159 245 245 245 F F F Max gen: 161, Max flush: 253 Begin loop 81 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:39.957750772Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:36:39.999469191Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:39.999833658Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.030854448Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.031421205Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.03145738Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.031471958Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.031486646Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835} {"msg":"Crucible d541a88b-9ec8-4b20-ad2c-ea355d58bf71 has session id: e3d8e01f-4cfe-4c41-a961-190a658e222f","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.031897156Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"Upstairs opts: Upstairs UUID: d541a88b-9ec8-4b20-ad2c-ea355d58bf71, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.031941969Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"Crucible stats registered with UUID: d541a88b-9ec8-4b20-ad2c-ea355d58bf71","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.031967146Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.06282329Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.062859095Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835} {"msg":"d541a88b-9ec8-4b20-ad2c-ea355d58bf71 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.062886111Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.062903849Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"client":"0","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.062932315Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"client":"0","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.062956791Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"client":"1","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.062978468Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"client":"1","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.063011783Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"client":"2","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.063046898Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"client":"2","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.063080093Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"":"io task","client":"2","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.063116848Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"":"io task","client":"0","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.063151992Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"":"io task","client":"1","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"ds_connection connected from Ok(127.0.0.1:63070)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.063188967Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"":"io task","client":"2","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"ds_connection connected from Ok(127.0.0.1:42828)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.063223102Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"":"io task","client":"1","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"ds_connection connected from Ok(127.0.0.1:39392)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.063258677Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"":"io task","client":"0","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.063953305Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"client":"2","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.063983611Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"client":"2","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.064000468Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"client":"1","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.064075488Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"client":"1","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.064101294Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"client":"0","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.064117502Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"client":"0","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.064641645Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"client":"2","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.064771416Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"client":"1","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.064793843Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"client":"0","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.065216401Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"client":"2","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.065246946Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"client":"2","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.065262174Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.065337043Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"client":"1","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.06535737Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"client":"1","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.065371398Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.065384006Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"client":"0","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.065397664Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"client":"0","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"[0]R flush_numbers[0..12]: [242, 238, 232, 252, 253, 228, 231, 253, 236, 252, 244, 240]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.065411922Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"":"downstairs","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"[0]R generation[0..12]: [158, 155, 150, 160, 161, 148, 149, 161, 153, 160, 158, 157]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.06542597Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"":"downstairs","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.065440098Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"":"downstairs","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"[1]R flush_numbers[0..12]: [242, 238, 232, 252, 253, 228, 231, 253, 236, 252, 244, 240]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.065454146Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"":"downstairs","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"[1]R generation[0..12]: [158, 155, 150, 160, 161, 148, 149, 161, 153, 160, 158, 157]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.065471244Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"":"downstairs","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.065485752Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"":"downstairs","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"[2]R flush_numbers[0..12]: [242, 238, 232, 252, 253, 228, 231, 253, 236, 252, 244, 240]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.06549963Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"":"downstairs","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"[2]R generation[0..12]: [158, 155, 150, 160, 161, 148, 149, 161, 153, 160, 158, 157]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.065513487Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"":"downstairs","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.065527476Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"":"downstairs","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.065540594Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"":"downstairs","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:36:40.065554082Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"":"downstairs","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"Next flush: 254","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.065576209Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"":"downstairs","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.065589427Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"":"downstairs","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.065602645Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.065614863Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"client":"0","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.065628251Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"client":"1","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.065641519Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"client":"2","":"downstairs","session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.065654927Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.065669355Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"d541a88b-9ec8-4b20-ad2c-ea355d58bf71 is now active with session: e3d8e01f-4cfe-4c41-a961-190a658e222f","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.065682423Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.065694571Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835,"session_id":"e3d8e01f-4cfe-4c41-a961-190a658e222f"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:40.065706869Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 308 len 12288 data: 10 10 11 03/30 Read block 404 len 8192 04/30 Write block 373 len 40960 data: 16 16 17 17 18 15 13 13 10 9 05/30 Read block 220 len 16384 06/30 Write block 19 len 4096 data: 15 07/30 Read block 315 len 24576 08/30 Write block 211 len 24576 data: 9 10 13 15 13 12 09/30 Write block 578 len 36864 data: 14 15 12 12 12 13 13 11 13 10/30 Write block 302 len 32768 data: 11 7 10 9 10 10 11 11 11/30 Write block 368 len 12288 data: 13 12 15 12/30 Read block 138 len 12288 13/30 Write block 233 len 4096 data: 15 14/30 Write block 505 len 24576 data: 10 10 11 13 11 11 15/30 Read block 338 len 20480 16/30 Read block 595 len 4096 17/30 Read block 557 len 24576 18/30 Read block 551 len 32768 19/30 Read block 403 len 36864 20/30 Flush 21/30 Write block 323 len 20480 data: 13 12 12 13 11 22/30 Read block 520 len 12288 23/30 Flush 24/30 Write block 76 len 4096 data: 18 25/30 Read block 35 len 16384 26/30 Write block 326 len 36864 data: 14 12 9 11 11 8 7 8 9 27/30 Read block 22 len 16384 28/30 Read block 167 len 32768 29/30 Write block 181 len 24576 data: 9 12 16 19 19 22 30/30 Write block 250 len 4096 data: 17 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 1 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:36:40.084526562Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1835} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 255 255 255 F F F 1 020-039 155 155 155 238 238 238 F F F 2 040-059 150 150 150 232 232 232 F F F 3 060-079 160 160 160 252 252 252 T T T 4 080-099 161 161 161 253 253 253 F F F 5 100-119 148 148 148 228 228 228 F F F 6 120-139 149 149 149 231 231 231 F F F 7 140-159 161 161 161 253 253 253 F F F 8 160-179 153 153 153 236 236 236 F F F 9 180-199 160 160 160 252 252 252 T T T 10 200-219 162 162 162 255 255 255 F F F 11 220-239 162 162 162 255 255 255 F F F 12 240-259 160 160 160 250 250 250 T T T 13 260-279 160 160 160 249 249 249 F F F 14 280-299 152 152 152 235 235 235 F F F 15 300-319 162 162 162 255 255 255 F F F 16 320-339 162 162 162 256 256 256 T T T 17 340-359 153 153 153 236 236 236 F F F 18 360-379 162 162 162 255 255 255 F F F 19 380-399 162 162 162 255 255 255 F F F 20 400-419 156 156 156 239 239 239 F F F 21 420-439 157 157 157 240 240 240 F F F 22 440-459 159 159 159 245 245 245 F F F 23 460-479 160 160 160 251 251 251 F F F 24 480-499 160 160 160 251 251 251 F F F 25 500-519 162 162 162 255 255 255 F F F 26 520-539 161 161 161 253 253 253 F F F 27 540-559 161 161 161 253 253 253 F F F 28 560-579 162 162 162 255 255 255 F F F 29 580-599 162 162 162 255 255 255 F F F Max gen: 162, Max flush: 256 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 255 255 255 F F F 1 020-039 155 155 155 238 238 238 F F F 2 040-059 150 150 150 232 232 232 F F F 3 060-079 163 163 163 257 257 257 F F F 4 080-099 161 161 161 253 253 253 F F F 5 100-119 148 148 148 228 228 228 F F F 6 120-139 149 149 149 231 231 231 F F F 7 140-159 161 161 161 253 253 253 F F F 8 160-179 153 153 153 236 236 236 F F F 9 180-199 163 163 163 257 257 257 F F F 10 200-219 162 162 162 255 255 255 F F F 11 220-239 162 162 162 255 255 255 F F F 12 240-259 163 163 163 257 257 257 F F F 13 260-279 160 160 160 249 249 249 F F F 14 280-299 152 152 152 235 235 235 F F F 15 300-319 162 162 162 255 255 255 F F F 16 320-339 163 163 163 257 257 257 F F F 17 340-359 153 153 153 236 236 236 F F F 18 360-379 162 162 162 255 255 255 F F F 19 380-399 162 162 162 255 255 255 F F F 20 400-419 156 156 156 239 239 239 F F F 21 420-439 157 157 157 240 240 240 F F F 22 440-459 159 159 159 245 245 245 F F F 23 460-479 160 160 160 251 251 251 F F F 24 480-499 160 160 160 251 251 251 F F F 25 500-519 162 162 162 255 255 255 F F F 26 520-539 161 161 161 253 253 253 F F F 27 540-559 161 161 161 253 253 253 F F F 28 560-579 162 162 162 255 255 255 F F F 29 580-599 162 162 162 255 255 255 F F F Max gen: 163, Max flush: 257 Begin loop 82 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.375407943Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:36:44.417725244Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.418084231Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.449707304Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.450314336Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.450357459Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.450374667Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.450392564Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843} {"msg":"Crucible 6b3817fd-c2c7-4e08-a021-9d1eb8c6db61 has session id: 902b32c0-55ff-48fa-a0f1-868bb713dc67","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.450767999Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"Upstairs opts: Upstairs UUID: 6b3817fd-c2c7-4e08-a021-9d1eb8c6db61, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.450821282Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"Crucible stats registered with UUID: 6b3817fd-c2c7-4e08-a021-9d1eb8c6db61","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.450851257Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.481363762Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.481394727Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843} {"msg":"6b3817fd-c2c7-4e08-a021-9d1eb8c6db61 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.481429132Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.48144596Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"client":"0","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.481474256Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"client":"0","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.481492373Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"client":"1","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.481509881Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"client":"1","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.481530238Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"client":"2","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.481547945Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"client":"2","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.481607256Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"":"io task","client":"2","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.481647461Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"":"io task","client":"0","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.481679096Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"":"io task","client":"1","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"ds_connection connected from Ok(127.0.0.1:62587)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.481707922Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"":"io task","client":"2","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"ds_connection connected from Ok(127.0.0.1:41231)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.481972543Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"":"io task","client":"0","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"ds_connection connected from Ok(127.0.0.1:58922)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.482033254Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"":"io task","client":"1","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.482449953Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"client":"2","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.482481279Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"client":"2","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.482509955Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"client":"1","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.48253894Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"client":"1","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.48260619Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"client":"0","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.482640385Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"client":"0","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.483127854Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"client":"1","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.483163879Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"client":"2","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.483180407Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"client":"0","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.48370267Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"client":"0","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.483732126Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"client":"0","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.483748104Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.483816544Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"client":"1","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.483836391Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"client":"1","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.483850439Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.483863067Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"client":"2","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.483876595Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"client":"2","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"[0]R flush_numbers[0..12]: [255, 238, 232, 257, 253, 228, 231, 253, 236, 257, 255, 255]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.483890623Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"":"downstairs","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"[0]R generation[0..12]: [162, 155, 150, 163, 161, 148, 149, 161, 153, 163, 162, 162]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.483904731Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"":"downstairs","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.483918929Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"":"downstairs","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"[1]R flush_numbers[0..12]: [255, 238, 232, 257, 253, 228, 231, 253, 236, 257, 255, 255]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.483932997Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"":"downstairs","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"[1]R generation[0..12]: [162, 155, 150, 163, 161, 148, 149, 161, 153, 163, 162, 162]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.483947794Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"":"downstairs","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.483965142Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"":"downstairs","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"[2]R flush_numbers[0..12]: [255, 238, 232, 257, 253, 228, 231, 253, 236, 257, 255, 255]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.48397952Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"":"downstairs","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"[2]R generation[0..12]: [162, 155, 150, 163, 161, 148, 149, 161, 153, 163, 162, 162]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.483993488Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"":"downstairs","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.484007636Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"":"downstairs","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.484020774Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"":"downstairs","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.484034142Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"":"downstairs","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"Next flush: 258","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.48404734Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"":"downstairs","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.484060528Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"":"downstairs","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.484074016Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,Wait for a query_work_queue command to finish before sending IO "session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.484094763Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"client":"0","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.484108081Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"client":"1","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.484121289Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"client":"2","":"downstairs","session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.484134867Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.484147285Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"6b3817fd-c2c7-4e08-a021-9d1eb8c6db61 is now active with session: 902b32c0-55ff-48fa-a0f1-868bb713dc67","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.484160263Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.484175111Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:44.48418665Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843,"session_id":"902b32c0-55ff-48fa-a0f1-868bb713dc67"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 432 len 40960 data: 8 6 7 10 12 8 11 10 11 13 02/30 Read block 308 len 20480 03/30 Write block 27 len 8192 data: 8 9 04/30 Read block 239 len 40960 05/30 Read block 332 len 4096 06/30 Read block 235 len 8192 07/30 Write block 382 len 20480 data: 10 7 8 6 7 08/30 Read block 578 len 4096 09/30 Write block 541 len 12288 data: 10 13 14 10/30 Read block 116 len 32768 11/30 Read block 537 len 4096 12/30 Write block 203 len 32768 data: 9 7 7 8 7 8 8 8 13/30 Read block 498 len 8192 14/30 Write block 513 len 24576 data: 10 13 9 11 9 11 15/30 Read block 190 len 32768 16/30 Flush 17/30 Write block 213 len 8192 data: 14 16 18/30 Write block 405 len 32768 data: 9 11 11 12 14 16 19 20 19/30 Write block 336 len 12288 data: 8 8 9 20/30 Write block 25 len 36864 data: 9 7 9 10 9 10 10 6 9 21/30 Read block 140 len 20480 22/30 Write block 292 len 40960 data: 10 10 10 9 9 9 6 7 8 9 23/30 Read block 202 len 24576 24/30 Write block 547 len 12288 data: 16 19 15 25/30 Write block 160 len 36864 data: 15 17 19 18 20 19 18 20 11 26/30 Write block 42 len 12288 data: 14 12 12 27/30 Write block 240 len 28672 data: 16 16 15 17 19 20 17 28/30 Read block 323 len 20480 29/30 Read block 539 len 28672 30/30 Write block 198 len 40960 data: 12 14 13 12 12 10 8 8 9 8 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 2 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:36:44.504226844Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1843} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 255 255 255 F F F 1 020-039 164 164 164 258 258 258 T T T 2 040-059 150 150 150 232 232 232 T T T 3 060-079 163 163 163 257 257 257 F F F 4 080-099 161 161 161 253 253 253 F F F 5 100-119 148 148 148 228 228 228 F F F 6 120-139 149 149 149 231 231 231 F F F 7 140-159 161 161 161 253 253 253 F F F 8 160-179 153 153 153 236 236 236 T T T 9 180-199 163 163 163 257 257 257 T T T 10 200-219 164 164 164 258 258 258 T T T 11 220-239 162 162 162 255 255 255 F F F 12 240-259 163 163 163 257 257 257 T T T 13 260-279 160 160 160 249 249 249 F F F 14 280-299 152 152 152 235 235 235 T T T 15 300-319 162 162 162 255 255 255 T T T 16 320-339 163 163 163 257 257 257 T T T 17 340-359 153 153 153 236 236 236 F F F 18 360-379 162 162 162 255 255 255 F F F 19 380-399 164 164 164 258 258 258 F F F 20 400-419 156 156 156 239 239 239 T T T 21 420-439 164 164 164 258 258 258 F F F 22 440-459 164 164 164 258 258 258 F F F 23 460-479 160 160 160 251 251 251 F F F 24 480-499 160 160 160 251 251 251 F F F 25 500-519 164 164 164 258 258 258 F F F 26 520-539 161 161 161 253 253 253 F F F 27 540-559 164 164 164 258 258 258 T T T 28 560-579 162 162 162 255 255 255 F F F 29 580-599 162 162 162 255 255 255 F F F Max gen: 164, Max flush: 258 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 255 255 255 F F F 1 020-039 165 165 165 259 259 259 F F F 2 040-059 165 165 165 259 259 259 F F F 3 060-079 163 163 163 257 257 257 F F F 4 080-099 161 161 161 253 253 253 F F F 5 100-119 148 148 148 228 228 228 F F F 6 120-139 149 149 149 231 231 231 F F F 7 140-159 161 161 161 253 253 253 F F F 8 160-179 165 165 165 259 259 259 F F F 9 180-199 165 165 165 259 259 259 F F F 10 200-219 165 165 165 259 259 259 F F F 11 220-239 162 162 162 255 255 255 F F F 12 240-259 165 165 165 259 259 259 F F F 13 260-279 160 160 160 249 249 249 F F F 14 280-299 165 165 165 259 259 259 F F F 15 300-319 165 165 165 259 259 259 F F F 16 320-339 165 165 165 259 259 259 F F F 17 340-359 153 153 153 236 236 236 F F F 18 360-379 162 162 162 255 255 255 F F F 19 380-399 164 164 164 258 258 258 F F F 20 400-419 165 165 165 259 259 259 F F F 21 420-439 164 164 164 258 258 258 F F F 22 440-459 164 164 164 258 258 258 F F F 23 460-479 160 160 160 251 251 251 F F F 24 480-499 160 160 160 251 251 251 F F F 25 500-519 164 164 164 258 258 258 F F F 26 520-539 161 161 161 253 253 253 F F F 27 540-559 165 165 165 259 259 259 F F F 28 560-579 162 162 162 255 255 255 F F F 29 580-599 162 162 162 255 255 255 F F F Max gen: 165, Max flush: 259 Begin loop 83 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.837341218Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:36:48.87978658Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.880191891Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.911688562Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.912272967Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.91231712Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.912336507Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.912355695Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851} {"msg":"Crucible 55c26c76-45b4-4838-a3f2-f0944f8e7290 has session id: 37ab923e-a7fd-4a25-b556-79483f4a1ff1","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.912800539Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"Upstairs opts: Upstairs UUID: 55c26c76-45b4-4838-a3f2-f0944f8e7290, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.912838254Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"Crucible stats registered with UUID: 55c26c76-45b4-4838-a3f2-f0944f8e7290","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.912855451Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.943492118Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.943528623Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851} {"msg":"55c26c76-45b4-4838-a3f2-f0944f8e7290 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.943555199Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.943571857Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"client":"0","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.943600432Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"client":"0","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.94361888Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"client":"1","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.943636507Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"client":"1","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.943658094Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"client":"2","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.943675981Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"client":"2","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.943693079Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"":"io task","client":"2","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.943711046Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"":"io task","client":"0","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.943729334Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"":"io task","client":"1","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"ds_connection connected from Ok(127.0.0.1:53447)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.943760259Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"":"io task","client":"0","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"ds_connection connected from Ok(127.0.0.1:46938)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.943924835Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"":"io task","client":"2","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"ds_connection connected from Ok(127.0.0.1:46869)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.944061375Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"":"io task","client":"1","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.944654918Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"client":"2","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.944696472Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"client":"2","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.944728218Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"client":"1","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.944759553Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"client":"1","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.944790928Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"client":"0","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.944809686Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"client":"0","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.945343118Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"client":"1","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.945377553Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"client":"2","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.94539487Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"client":"0","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.945921583Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"client":"2","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.945951829Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"client":"2","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.945967897Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.94601549Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"client":"1","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.946043876Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"client":"1","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.946058963Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.946071492Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"client":"0","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.94608517Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"client":"0","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"[0]R flush_numbers[0..12]: [255, 259, 259, 257, 253, 228, 231, 253, 259, 259, 259, 255]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.946099208Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"":"downstairs","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"[0]R generation[0..12]: [162, 165, 165, 163, 161, 148, 149, 161, 165, 165, 165, 162]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.946114055Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"":"downstairs","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.946128293Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"":"downstairs","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"[1]R flush_numbers[0..12]: [255, 259, 259, 257, 253, 228, 231, 253, 259, 259, 259, 255]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.946142371Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"":"downstairs","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"[1]R generation[0..12]: [162, 165, 165, 163, 161, 148, 149, 161, 165, 165, 165, 162]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.946156139Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"":"downstairs","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.946173457Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"":"downstairs","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"[2]R flush_numbers[0..12]: [255, 259, 259, 257, 253, 228, 231, 253, 259, 259, 259, 255]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.946187585Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"":"downstairs","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"[2]R generation[0..12]: [162, 165, 165, 163, 161, 148, 149, 161, 165, 165, 165, 162]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.946201462Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"":"downstairs","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.946215631Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"":"downstairs","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-01-03T08:36:48.946228898Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"":"downstairs","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.946251445Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"":"downstairs","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"Next flush: 260","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.946264823Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"":"downstairs","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.946278111Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"":"downstairs","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.94629166Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.946303838Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"client":"0","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.946317276Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"client":"1","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.946330594Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"client":"2","":"downstairs","session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.946343882Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.94635616Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"55c26c76-45b4-4838-a3f2-f0944f8e7290 is now active with session: 37ab923e-a7fd-4a25-b556-79483f4a1ff1","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.946369078Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.946382836Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851,"session_id":"37ab923e-a7fd-4a25-b556-79483f4a1ff1"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:48.946395504Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 227 len 8192 data: 14 17 03/30 Read block 0 len 28672 04/30 Read block 408 len 12288 05/30 Write block 488 len 24576 data: 10 12 10 10 12 14 06/30 Write block 39 len 4096 data: 13 07/30 Write block 192 len 4096 data: 12 08/30 Flush 09/30 Flush 10/30 Write block 569 len 12288 data: 13 12 12 11/30 Read block 546 len 36864 12/30 Write block 397 len 40960 data: 13 14 14 13 12 11 9 8 10 12 13/30 Read block 508 len 40960 14/30 Read block 216 len 4096 15/30 Read block 382 len 20480 16/30 Write block 133 len 4096 data: 12 17/30 Write block 186 len 8192 data: 23 19 18/30 Flush 19/30 Write block 299 len 8192 data: 8 9 20/30 Write block 241 len 36864 data: 17 16 18 20 21 18 18 18 16 21/30 Read block 369 len 40960 22/30 Write block 76 len 20480 data: 19 17 13 13 14 23/30 Read block 151 len 12288 24/30 Read block 351 len 12288 25/30 Write block 175 len 12288 data: 11 13 13 26/30 Read block 66 len 20480 27/30 Write block 222 len 8192 data: 14 13 28/30 Write block 249 len 32768 data: 17 18 17 19 20 21 16 20 29/30 Write block 257 len 16384 data: 19 19 17 17 30/30 Write block 194 len 28672 data: 11 11 13 13 13 15 14 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 2 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:36:48.964113137Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1851} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 255 255 255 F F F 1 020-039 166 166 166 261 261 261 F F F 2 040-059 165 165 165 259 259 259 F F F 3 060-079 163 163 163 257 257 257 T T T 4 080-099 161 161 161 253 253 253 T T T 5 100-119 148 148 148 228 228 228 F F F 6 120-139 166 166 166 263 263 263 F F F 7 140-159 161 161 161 253 253 253 F F F 8 160-179 165 165 165 259 259 259 T T T 9 180-199 166 166 166 263 263 263 T T T 10 200-219 165 165 165 259 259 259 T T T 11 220-239 166 166 166 261 261 261 T T T 12 240-259 165 165 165 259 259 259 T T T 13 260-279 160 160 160 249 249 249 T T T 14 280-299 165 165 165 259 259 259 T T T 15 300-319 165 165 165 259 259 259 T T T 16 320-339 165 165 165 259 259 259 F F F 17 340-359 153 153 153 236 236 236 F F F 18 360-379 162 162 162 255 255 255 F F F 19 380-399 166 166 166 263 263 263 F F F 20 400-419 166 166 166 263 263 263 F F F 21 420-439 164 164 164 258 258 258 F F F 22 440-459 164 164 164 258 258 258 F F F 23 460-479 160 160 160 251 251 251 F F F 24 480-499 166 166 166 261 261 261 F F F 25 500-519 164 164 164 258 258 258 F F F 26 520-539 161 161 161 253 253 253 F F F 27 540-559 165 165 165 259 259 259 F F F 28 560-579 166 166 166 263 263 263 F F F 29 580-599 162 162 162 255 255 255 F F F Max gen: 166, Max flush: 263 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 255 255 255 F F F 1 020-039 166 166 166 261 261 261 F F F 2 040-059 165 165 165 259 259 259 F F F 3 060-079 167 167 167 264 264 264 F F F 4 080-099 167 167 167 264 264 264 F F F 5 100-119 148 148 148 228 228 228 F F F 6 120-139 166 166 166 263 263 263 F F F 7 140-159 161 161 161 253 253 253 F F F 8 160-179 167 167 167 264 264 264 F F F 9 180-199 167 167 167 264 264 264 F F F 10 200-219 167 167 167 264 264 264 F F F 11 220-239 167 167 167 264 264 264 F F F 12 240-259 167 167 167 264 264 264 F F F 13 260-279 167 167 167 264 264 264 F F F 14 280-299 167 167 167 264 264 264 F F F 15 300-319 167 167 167 264 264 264 F F F 16 320-339 165 165 165 259 259 259 F F F 17 340-359 153 153 153 236 236 236 F F F 18 360-379 162 162 162 255 255 255 F F F 19 380-399 166 166 166 263 263 263 F F F 20 400-419 166 166 166 263 263 263 F F F 21 420-439 164 164 164 258 258 258 F F F 22 440-459 164 164 164 258 258 258 F F F 23 460-479 160 160 160 251 251 251 F F F 24 480-499 166 166 166 261 261 261 F F F 25 500-519 164 164 164 258 258 258 F F F 26 520-539 161 161 161 253 253 253 F F F 27 540-559 165 165 165 259 259 259 F F F 28 560-579 166 166 166 263 263 263 F F F 29 580-599 162 162 162 255 255 255 F F F Max gen: 167, Max flush: 264 Begin loop 84 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.301441727Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.34456383Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.345134587Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.345171422Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.345190769Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.345205597Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859} {"msg":"Crucible c593ddf1-f1c0-47e9-bb0d-7636b9566861 has session id: 035e4f4d-d479-48fa-b401-ab334fdf0e4e","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.345642593Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"Upstairs opts: Upstairs UUID: c593ddf1-f1c0-47e9-bb0d-7636b9566861, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.345686326Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"Crucible stats registered with UUID: c593ddf1-f1c0-47e9-bb0d-7636b9566861","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.345711993Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.376557339Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.376590704Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859} {"msg":"c593ddf1-f1c0-47e9-bb0d-7636b9566861 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.376624839Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.376642596Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"client":"0","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.376671902Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"client":"0","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.376690169Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"client":"1","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.376707537Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"client":"1","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.376724634Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"client":"2","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.376742242Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"client":"2","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.376763779Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"":"io task","client":"2","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.376779016Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"":"io task","client":"0","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.376792924Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"":"io task","client":"1","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:47352)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.37695691Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"":"io task","client":"1","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:64806)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.376998334Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"":"io task","client":"2","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:37415)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.377212913Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"":"io task","client":"0","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.377741936Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"client":"0","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.377772002Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"client":"0","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.377787329Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"client":"2","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.377801577Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"client":"2","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.377815255Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"client":"1","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.377867528Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"client":"1","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.378391851Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"client":"1","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.378422067Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"client":"0","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.378451062Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"client":"2","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.378993163Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"client":"2","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.379020799Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"client":"2","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.379036587Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.379128663Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"client":"1","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.37914907Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"client":"1","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.379163108Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.379175517Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"client":"0","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.379188884Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"client":"0","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"[0]R flush_numbers[0..12]: [255, 261, 259, 264, 264, 228, 263, 253, 264, 264, 264, 264]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.379202822Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"":"downstairs","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"[0]R generation[0..12]: [162, 166, 165, 167, 167, 148, 166, 161, 167, 167, 167, 167]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.379216871Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"":"downstairs","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.379230998Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"":"downstairs","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"[1]R flush_numbers[0..12]: [255, 261, 259, 264, 264, 228, 263, 253, 264, 264, 264, 264]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.379244876Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"":"downstairs","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"[1]R generation[0..12]: [162, 166, 165, 167, 167, 148, 166, 161, 167, 167, 167, 167]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.379258694Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"":"downstairs","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.379272852Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"":"downstairs","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"[2]R flush_numbers[0..12]: [255, 261, 259, 264, 264, 228, 263, 253, 264, 264, 264, 264]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.37928646Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"":"downstairs","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"[2]R generation[0..12]: [162, 166, 165, 167, 167, 148, 166, 161, 167, 167, 167, 167]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.379304728Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"":"downstairs","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.379319165Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"":"downstairs","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2025-01-03T08:36:53.379332244Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"":"downstairs","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.379353691Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"":"downstairs","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"Next flush: 265","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.379366898Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"":"downstairs","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.379380017Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"":"downstairs","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.379393305Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.379405583Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"client":"0","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.379418831Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"client":"1","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.379432039Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"client":"2","":"downstairs","session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.379445217Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.379457515Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"c593ddf1-f1c0-47e9-bb0d-7636b9566861 is now active with session: 035e4f4d-d479-48fa-b401-ab334fdf0e4e","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.379470383Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.379482652Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859,"session_id":"035e4f4d-d479-48fa-b401-ab334fdf0e4e"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:53.37949495Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 373 len 40960 data: 17 17 18 18 19 16 14 14 11 11 02/30 Write block 271 len 40960 data: 12 11 13 11 12 10 10 9 6 7 03/30 Read block 258 len 24576 04/30 Write block 242 len 28672 data: 17 19 21 22 19 19 19 05/30 Read block 338 len 8192 06/30 Write block 408 len 28672 data: 13 15 17 20 21 20 23 07/30 Read block 111 len 28672 08/30 Read block 89 len 36864 09/30 Read block 241 len 20480 10/30 Write block 567 len 4096 data: 12 11/30 Read block 406 len 36864 12/30 Write block 24 len 12288 data: 11 10 8 13/30 Read block 416 len 20480 14/30 Write block 498 len 36864 data: 13 15 15 15 11 12 11 11 11 15/30 Read block 112 len 16384 16/30 Read block 71 len 32768 17/30 Write block 487 len 20480 data: 13 11 13 11 11 18/30 Write block 30 len 32768 data: 11 11 7 10 9 5 6 9 19/30 Read block 197 len 16384 20/30 Read block 101 len 8192 21/30 Read block 29 len 24576 22/30 Write block 537 len 16384 data: 8 7 7 8 23/30 Write block 127 len 12288 data: 13 12 16 24/30 Write block 379 len 8192 data: 15 15 25/30 Read block 348 len 20480 26/30 Read block 563 len 32768 27/30 Read block 156 len 8192 28/30 Read block 400 len 40960 29/30 Read block 495 len 36864 30/30 Read block 207 len 24576 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Read 6 Done Done Done false 1003 Acked Write 7 Done Done Done false 1004 Acked Read 2 Done Done Done false 1005 Acked Write 7 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Read 9 Done Done Done false 1008 Acked Read 5 Done Done Done false 1009 Acked Write 1 Done Done Done false 1010 Acked Read 9 Done Done Done false 1011 Acked Write 3 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Write 9 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 6 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:36:53.399632999Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1859} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 255 255 255 F F F 1 020-039 166 166 166 261 261 261 T T T 2 040-059 165 165 165 259 259 259 F F F 3 060-079 167 167 167 264 264 264 F F F 4 080-099 167 167 167 264 264 264 F F F 5 100-119 148 148 148 228 228 228 F F F 6 120-139 166 166 166 263 263 263 T T T 7 140-159 161 161 161 253 253 253 F F F 8 160-179 167 167 167 264 264 264 F F F 9 180-199 167 167 167 264 264 264 F F F 10 200-219 167 167 167 264 264 264 F F F 11 220-239 167 167 167 264 264 264 F F F 12 240-259 167 167 167 264 264 264 T T T 13 260-279 167 167 167 264 264 264 T T T 14 280-299 167 167 167 264 264 264 T T T 15 300-319 167 167 167 264 264 264 F F F 16 320-339 165 165 165 259 259 259 F F F 17 340-359 153 153 153 236 236 236 F F F 18 360-379 162 162 162 255 255 255 T T T 19 380-399 166 166 166 263 263 263 T T T 20 400-419 166 166 166 263 263 263 T T T 21 420-439 164 164 164 258 258 258 F F F 22 440-459 164 164 164 258 258 258 F F F 23 460-479 160 160 160 251 251 251 F F F 24 480-499 166 166 166 261 261 261 T T T 25 500-519 164 164 164 258 258 258 T T T 26 520-539 161 161 161 253 253 253 T T T 27 540-559 165 165 165 259 259 259 T T T 28 560-579 166 166 166 263 263 263 T T T 29 580-599 162 162 162 255 255 255 F F F Max gen: 167, Max flush: 264 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 255 255 255 F F F 1 020-039 168 168 168 265 265 265 F F F 2 040-059 165 165 165 259 259 259 F F F 3 060-079 167 167 167 264 264 264 F F F 4 080-099 167 167 167 264 264 264 F F F 5 100-119 148 148 148 228 228 228 F F F 6 120-139 168 168 168 265 265 265 F F F 7 140-159 161 161 161 253 253 253 F F F 8 160-179 167 167 167 264 264 264 F F F 9 180-199 167 167 167 264 264 264 F F F 10 200-219 167 167 167 264 264 264 F F F 11 220-239 167 167 167 264 264 264 F F F 12 240-259 168 168 168 265 265 265 F F F 13 260-279 168 168 168 265 265 265 F F F 14 280-299 168 168 168 265 265 265 F F F 15 300-319 167 167 167 264 264 264 F F F 16 320-339 165 165 165 259 259 259 F F F 17 340-359 153 153 153 236 236 236 F F F 18 360-379 168 168 168 265 265 265 F F F 19 380-399 168 168 168 265 265 265 F F F 20 400-419 168 168 168 265 265 265 F F F 21 420-439 164 164 164 258 258 258 F F F 22 440-459 164 164 164 258 258 258 F F F 23 460-479 160 160 160 251 251 251 F F F 24 480-499 168 168 168 265 265 265 F F F 25 500-519 168 168 168 265 265 265 F F F 26 520-539 168 168 168 265 265 265 F F F 27 540-559 168 168 168 265 265 265 F F F 28 560-579 168 168 168 265 265 265 F F F 29 580-599 162 162 162 255 255 255 F F F Max gen: 168, Max flush: 265 Begin loop 85 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.714446136Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:36:57.757172738Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.757537504Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.789609971Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.790267105Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.790309369Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.790339974Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.790359631Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867} {"msg":"Crucible 731d9932-5854-4896-931d-a1de49bc6658 has session id: dc02f02a-51e9-40ed-ac05-34f2ae6d739f","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.79077913Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"Upstairs opts: Upstairs UUID: 731d9932-5854-4896-931d-a1de49bc6658, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.790825064Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"Crucible stats registered with UUID: 731d9932-5854-4896-931d-a1de49bc6658","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.79084829Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.821572914Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.821606369Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867} {"msg":"731d9932-5854-4896-931d-a1de49bc6658 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.821640404Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.821657722Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"client":"0","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.821685248Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"client":"0","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.821703385Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"client":"1","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.821725052Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"client":"1","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.821746459Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"client":"2","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.821778584Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"client":"2","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.821810869Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"":"io task","client":"2","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.821842665Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"":"io task","client":"0","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.821869181Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"":"io task","client":"1","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"ds_connection connected from Ok(127.0.0.1:42263)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.821906955Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"":"io task","client":"2","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"ds_connection connected from Ok(127.0.0.1:41979)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.82194218Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"":"io task","client":"0","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"ds_connection connected from Ok(127.0.0.1:43131)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.821996362Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"":"io task","client":"1","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.822667434Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"client":"2","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.822706208Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"client":"2","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.822777918Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"client":"1","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.822823051Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"client":"1","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.822879603Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"client":"0","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.822909809Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"client":"0","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.823351894Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"client":"1","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.8233811Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"client":"2","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.823397978Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"client":"0","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.823940658Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"client":"1","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.823981272Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"client":"1","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.824007749Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.824021647Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"client":"0","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.824035674Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"client":"0","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.824049312Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.824061611Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"client":"2","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.824075299Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"client":"2","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"[0]R flush_numbers[0..12]: [255, 265, 259, 264, 264, 228, 265, 253, 264, 264, 264, 264]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.824089326Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"":"downstairs","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"[0]R generation[0..12]: [162, 168, 165, 167, 167, 148, 168, 161, 167, 167, 167, 167]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.824103495Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"":"downstairs","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.824117673Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"":"downstairs","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"[1]R flush_numbers[0..12]: [255, 265, 259, 264, 264, 228, 265, 253, 264, 264, 264, 264]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.82413177Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"":"downstairs","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"[1]R generation[0..12]: [162, 168, 165, 167, 167, 148, 168, 161, 167, 167, 167, 167]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.824150428Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"":"downstairs","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.824165255Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"[2]R flush_numbers[0..12]: [255, 265, 259, 264, 264, 228, 265, 253, 264, 264, 264, 264]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.824187402Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"":"downstairs","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"[2]R generation[0..12]: [162, 168, 165, 167, 167, 148, 168, 161, 167, 167, 167, 167]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.82420121Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"":"downstairs","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.824215428Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"":"downstairs","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"Max found gen is 169","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.824228516Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"":"downstairs","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"Generation requested: 170 >= found:169","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.824249773Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"":"downstairs","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"Next flush: 266","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.82427433Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"":"downstairs","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"All extents match","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-03T08:36:57.824291957Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"":"downstairs","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.824320673Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.824335561Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"client":"0","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.824349339Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"client":"1","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.824362507Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"client":"2","":"downstairs","session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.824376125Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.824392142Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"731d9932-5854-4896-931d-a1de49bc6658 is now active with session: dc02f02a-51e9-40ed-ac05-34f2ae6d739f","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.8244056Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.824417849Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867,"session_id":"dc02f02a-51e9-40ed-ac05-34f2ae6d739f"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:36:57.824430487Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 95 len 4096 data: 12 03/30 Read block 340 len 36864 04/30 Write block 19 len 40960 data: 16 13 14 11 10 12 11 9 10 11 05/30 Read block 77 len 36864 06/30 Read block 301 len 36864 07/30 Read block 518 len 12288 08/30 Read block 195 len 36864 09/30 Write block 522 len 8192 data: 13 14 10/30 Write block 208 len 36864 data: 9 9 9 10 11 15 17 14 13 11/30 Write block 588 len 16384 data: 14 17 16 13 12/30 Read block 405 len 4096 13/30 Flush 14/30 Write block 465 len 20480 data: 9 11 12 11 13 15/30 Write block 282 len 32768 data: 7 12 9 11 11 11 11 12 16/30 Read block 234 len 8192 17/30 Read block 350 len 20480 18/30 Read block 491 len 40960 19/30 Flush 20/30 Write block 376 len 24576 data: 19 20 17 16 16 12 21/30 Write block 120 len 20480 data: 18 16 16 19 16 22/30 Read block 1 len 36864 23/30 Read block 287 len 16384 24/30 Read block 179 len 8192 25/30 Read block 401 len 32768 26/30 Read block 468 len 28672 27/30 Write block 577 len 32768 data: 12 15 16 13 13 13 14 14 28/30 Write block 241 len 40960 data: 18 18 20 22 23 20 20 20 18 19 29/30 Write block 368 len 24576 data: 14 13 16 17 18 18 30/30 Write block 91 len 4096 data: 16 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 6 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:36:57.843225483Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1867} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 267 267 267 F F F 1 020-039 170 170 170 267 267 267 F F F 2 040-059 165 165 165 259 259 259 F F F 3 060-079 167 167 167 264 264 264 F F F 4 080-099 170 170 170 267 267 267 T T T 5 100-119 148 148 148 228 228 228 F F F 6 120-139 168 168 168 265 265 265 T T T 7 140-159 161 161 161 253 253 253 F F F 8 160-179 167 167 167 264 264 264 F F F 9 180-199 167 167 167 264 264 264 F F F 10 200-219 170 170 170 267 267 267 F F F 11 220-239 167 167 167 264 264 264 F F F 12 240-259 168 168 168 265 265 265 T T T 13 260-279 168 168 168 265 265 265 F F F 14 280-299 170 170 170 268 268 268 F F F 15 300-319 167 167 167 264 264 264 F F F 16 320-339 165 165 165 259 259 259 F F F 17 340-359 153 153 153 236 236 236 F F F 18 360-379 168 168 168 265 265 265 T T T 19 380-399 168 168 168 265 265 265 T T T 20 400-419 168 168 168 265 265 265 F F F 21 420-439 164 164 164 258 258 258 F F F 22 440-459 164 164 164 258 258 258 F F F 23 460-479 170 170 170 268 268 268 F F F 24 480-499 168 168 168 265 265 265 F F F 25 500-519 168 168 168 265 265 265 F F F 26 520-539 170 170 170 267 267 267 F F F 27 540-559 168 168 168 265 265 265 F F F 28 560-579 168 168 168 265 265 265 T T T 29 580-599 170 170 170 267 267 267 T T T Max gen: 170, Max flush: 268 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 267 267 267 F F F 1 020-039 170 170 170 267 267 267 F F F 2 040-059 165 165 165 259 259 259 F F F 3 060-079 167 167 167 264 264 264 F F F 4 080-099 171 171 171 269 269 269 F F F 5 100-119 148 148 148 228 228 228 F F F 6 120-139 171 171 171 269 269 269 F F F 7 140-159 161 161 161 253 253 253 F F F 8 160-179 167 167 167 264 264 264 F F F 9 180-199 167 167 167 264 264 264 F F F 10 200-219 170 170 170 267 267 267 F F F 11 220-239 167 167 167 264 264 264 F F F 12 240-259 171 171 171 269 269 269 F F F 13 260-279 168 168 168 265 265 265 F F F 14 280-299 170 170 170 268 268 268 F F F 15 300-319 167 167 167 264 264 264 F F F 16 320-339 165 165 165 259 259 259 F F F 17 340-359 153 153 153 236 236 236 F F F 18 360-379 171 171 171 269 269 269 F F F 19 380-399 171 171 171 269 269 269 F F F 20 400-419 168 168 168 265 265 265 F F F 21 420-439 164 164 164 258 258 258 F F F 22 440-459 164 164 164 258 258 258 F F F 23 460-479 170 170 170 268 268 268 F F F 24 480-499 168 168 168 265 265 265 F F F 25 500-519 168 168 168 265 265 265 F F F 26 520-539 170 170 170 267 267 267 F F F 27 540-559 168 168 168 265 265 265 F F F 28 560-579 171 171 171 269 269 269 F F F 29 580-599 171 171 171 269 269 269 F F F Max gen: 171, Max flush: 269 Begin loop 86 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.157023128Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.200775839Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.201426254Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.201465279Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.201480166Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.201494714Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875} {"msg":"Crucible f4a026be-a6ce-418c-948c-89cadb3790c7 has session id: e428d297-b458-4243-9711-bf0c428db7c9","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.201891606Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"Upstairs opts: Upstairs UUID: f4a026be-a6ce-418c-948c-89cadb3790c7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.20193704Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"Crucible stats registered with UUID: f4a026be-a6ce-418c-948c-89cadb3790c7","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.201960946Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.232659484Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.232692069Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875} {"msg":"f4a026be-a6ce-418c-948c-89cadb3790c7 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.232727964Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.232745131Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"client":"0","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.232772607Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"client":"0","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.232790585Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"client":"1","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.232807902Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"client":"1","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.23282496Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"client":"2","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.232846497Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"client":"2","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.232863544Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"":"io task","client":"2","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.232881392Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"":"io task","client":"0","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.232902568Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"":"io task","client":"1","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:48781)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.232952941Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"":"io task","client":"0","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:40060)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.232988476Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"":"io task","client":"2","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:45623)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.233021521Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"":"io task","client":"1","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.233775901Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"client":"0","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.233815025Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"client":"0","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.233844381Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"client":"1","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.233871217Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"client":"1","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.233938367Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"client":"2","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.233958844Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"client":"2","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.234507304Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"client":"1","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.234538439Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"client":"2","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.234558907Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"client":"0","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.235113275Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"client":"1","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.23514908Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"client":"1","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.235168467Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.235184695Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"client":"2","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.235201723Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"client":"2","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.23521898Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.235234758Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"client":"0","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.235252135Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"client":"0","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"[0]R flush_numbers[0..12]: [267, 267, 259, 264, 269, 228, 269, 253, 264, 264, 267, 264]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.235269523Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"":"downstairs","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"[0]R generation[0..12]: [170, 170, 165, 167, 171, 148, 171, 161, 167, 167, 170, 167]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.23528699Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"":"downstairs","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.235304518Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"":"downstairs","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"[1]R flush_numbers[0..12]: [267, 267, 259, 264, 269, 228, 269, 253, 264, 264, 267, 264]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.235321915Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"":"downstairs","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"[1]R generation[0..12]: [170, 170, 165, 167, 171, 148, 171, 161, 167, 167, 170, 167]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.235339873Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"":"downstairs","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {Wait for a query_work_queue command to finish before sending IO "msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.235365389Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"":"downstairs","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"[2]R flush_numbers[0..12]: [267, 267, 259, 264, 269, 228, 269, 253, 264, 264, 267, 264]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.235394814Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"":"downstairs","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"[2]R generation[0..12]: [170, 170, 165, 167, 171, 148, 171, 161, 167, 167, 170, 167]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.235414791Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"":"downstairs","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.235432619Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"":"downstairs","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.235449257Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"":"downstairs","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.235466054Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"":"downstairs","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"Next flush: 270","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.235482412Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"":"downstairs","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.235498799Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"":"downstairs","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.235515347Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.235530555Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"client":"0","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.235547132Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"client":"1","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.2355643Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"client":"2","":"downstairs","session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.235581957Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.235594435Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"f4a026be-a6ce-418c-948c-89cadb3790c7 is now active with session: e428d297-b458-4243-9711-bf0c428db7c9","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.235607503Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.235619812Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875,"session_id":"e428d297-b458-4243-9711-bf0c428db7c9"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:02.23563429Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 107 len 24576 data: 11 12 10 6 7 10 02/30 Read block 303 len 36864 03/30 Read block 530 len 36864 04/30 Write block 124 len 24576 data: 17 14 11 14 13 17 05/30 Read block 420 len 12288 06/30 Read block 437 len 36864 07/30 Read block 484 len 4096 08/30 Write block 247 len 16384 data: 21 21 19 20 09/30 Write block 427 len 8192 data: 9 9 10/30 Write block 218 len 4096 data: 12 11/30 Flush 12/30 Write block 293 len 32768 data: 11 11 10 10 10 7 9 10 13/30 Write block 243 len 16384 data: 21 23 24 21 14/30 Write block 313 len 8192 data: 13 15 15/30 Write block 548 len 8192 data: 20 16 16/30 Write block 409 len 16384 data: 16 18 21 22 17/30 Read block 523 len 8192 18/30 Read block 482 len 12288 19/30 Write block 158 len 24576 data: 14 13 16 18 20 19 20/30 Flush 21/30 Write block 35 len 24576 data: 6 7 10 9 14 12 22/30 Read block 91 len 20480 23/30 Read block 433 len 40960 24/30 Flush 25/30 Write block 163 len 8192 data: 20 21 26/30 Read block 45 len 36864 27/30 Write block 224 len 16384 data: 14 11 12 15 28/30 Read block 369 len 4096 29/30 Write block 555 len 20480 data: 13 10 9 10 8 30/30 Write block 115 len 12288 data: 12 16 19 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 2 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:02.253654568Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1875} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 267 267 267 F F F 1 020-039 172 172 172 272 272 272 F F F 2 040-059 172 172 172 272 272 272 F F F 3 060-079 167 167 167 264 264 264 F F F 4 080-099 171 171 171 269 269 269 F F F 5 100-119 172 172 172 270 270 270 T T T 6 120-139 172 172 172 270 270 270 F F F 7 140-159 172 172 172 271 271 271 F F F 8 160-179 172 172 172 271 271 271 T T T 9 180-199 167 167 167 264 264 264 F F F 10 200-219 172 172 172 270 270 270 F F F 11 220-239 167 167 167 264 264 264 T T T 12 240-259 172 172 172 271 271 271 F F F 13 260-279 168 168 168 265 265 265 F F F 14 280-299 172 172 172 271 271 271 F F F 15 300-319 172 172 172 271 271 271 F F F 16 320-339 165 165 165 259 259 259 F F F 17 340-359 153 153 153 236 236 236 F F F 18 360-379 171 171 171 269 269 269 F F F 19 380-399 171 171 171 269 269 269 F F F 20 400-419 172 172 172 271 271 271 F F F 21 420-439 172 172 172 270 270 270 F F F 22 440-459 164 164 164 258 258 258 F F F 23 460-479 170 170 170 268 268 268 F F F 24 480-499 168 168 168 265 265 265 F F F 25 500-519 168 168 168 265 265 265 F F F 26 520-539 170 170 170 267 267 267 F F F 27 540-559 172 172 172 271 271 271 T T T 28 560-579 171 171 171 269 269 269 F F F 29 580-599 171 171 171 269 269 269 F F F Max gen: 172, Max flush: 272 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 267 267 267 F F F 1 020-039 172 172 172 272 272 272 F F F 2 040-059 172 172 172 272 272 272 F F F 3 060-079 167 167 167 264 264 264 F F F 4 080-099 171 171 171 269 269 269 F F F 5 100-119 173 173 173 273 273 273 F F F 6 120-139 172 172 172 270 270 270 F F F 7 140-159 172 172 172 271 271 271 F F F 8 160-179 173 173 173 273 273 273 F F F 9 180-199 167 167 167 264 264 264 F F F 10 200-219 172 172 172 270 270 270 F F F 11 220-239 173 173 173 273 273 273 F F F 12 240-259 172 172 172 271 271 271 F F F 13 260-279 168 168 168 265 265 265 F F F 14 280-299 172 172 172 271 271 271 F F F 15 300-319 172 172 172 271 271 271 F F F 16 320-339 165 165 165 259 259 259 F F F 17 340-359 153 153 153 236 236 236 F F F 18 360-379 171 171 171 269 269 269 F F F 19 380-399 171 171 171 269 269 269 F F F 20 400-419 172 172 172 271 271 271 F F F 21 420-439 172 172 172 270 270 270 F F F 22 440-459 164 164 164 258 258 258 F F F 23 460-479 170 170 170 268 268 268 F F F 24 480-499 168 168 168 265 265 265 F F F 25 500-519 168 168 168 265 265 265 F F F 26 520-539 170 170 170 267 267 267 F F F 27 540-559 173 173 173 273 273 273 F F F 28 560-579 171 171 171 269 269 269 F F F 29 580-599 171 171 171 269 269 269 F F F Max gen: 173, Max flush: 273 Begin loop 87 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.514983454Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.558672285Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.559241062Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.559277866Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.559292824Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.559307732Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883} {"msg":"Crucible 2cd44144-50b3-4731-b7c5-07f0adfaac2d has session id: 17fb632b-7fb7-4dd8-ad0c-324dbc521c7a","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.559771814Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"Upstairs opts: Upstairs UUID: 2cd44144-50b3-4731-b7c5-07f0adfaac2d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.559808609Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"Crucible stats registered with UUID: 2cd44144-50b3-4731-b7c5-07f0adfaac2d","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.559824807Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.590795445Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.59082777Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883} {"msg":"2cd44144-50b3-4731-b7c5-07f0adfaac2d active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.590858585Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.590875683Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"client":"0","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.590902939Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"client":"0","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.590922696Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"client":"1","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.590940303Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"client":"1","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.590972019Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"client":"2","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.591006754Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"client":"2","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.591036489Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"":"io task","client":"2","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.591072244Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"":"io task","client":"0","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.591109429Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"":"io task","client":"1","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"ds_connection connected from Ok(127.0.0.1:50800)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.591154792Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"":"io task","client":"0","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"ds_connection connected from Ok(127.0.0.1:48637)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.591285373Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"":"io task","client":"2","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"ds_connection connected from Ok(127.0.0.1:52143)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.591319368Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"":"io task","client":"1","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.591916531Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"client":"1","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.591958125Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"client":"1","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.59199098Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"client":"2","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.592023685Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"client":"2","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.592054531Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"client":"0","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.592074868Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"client":"0","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.592573955Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"client":"0","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.592728152Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"client":"1","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.592758908Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"client":"2","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.593142162Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"client":"0","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.593183976Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"client":"0","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.593210812Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.593353631Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"client":"1","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.593376708Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"client":"1","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.593392156Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.593404863Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"client":"2","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.593418492Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"client":"2","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"[0]R flush_numbers[0..12]: [267, 272, 272, 264, 269, 273, 270, 271, 273, 264, 270, 273]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.59343243Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"":"downstairs","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"[0]R generation[0..12]: [170, 172, 172, 167, 171, 173, 172, 172, 173, 167, 172, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.593446708Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"":"downstairs","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.593461125Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"":"downstairs","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"[1]R flush_numbers[0..12]: [267, 272, 272, 264, 269, 273, 270, 271, 273, 264, 270, 273]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.593475143Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"":"downstairs","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"[1]R generation[0..12]: [170, 172, 172, 167, 171, 173, 172, 172, 173, 167, 172, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.593489001Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"":"downstairs","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.593504229Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"":"downstairs","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"[2]R flush_numbers[0..12]: [267, 272, 272, 264, 269, 273, 270, 271, 273, 264, 270, 273]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.593518557Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"":"downstairs","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"[2]R generation[0..12]: [170, 172, 172, 167, 171, 173, 172, 172, 173, 167, 172, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.593532575Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"":"downstairs","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.593546673Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"":"downstairs","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.593562281Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"":"downstairs","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"Generation requested: 174 >= found:174","Wait for a query_work_queue command to finish before sending IO v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.593583417Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"":"downstairs","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"Next flush: 274","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.593596655Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"":"downstairs","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.593609764Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"":"downstairs","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.593623142Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.59363551Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"client":"0","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.593649098Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"client":"1","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.593662296Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"client":"2","":"downstairs","session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.593675504Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.593687912Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"2cd44144-50b3-4731-b7c5-07f0adfaac2d is now active with session: 17fb632b-7fb7-4dd8-ad0c-324dbc521c7a","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.59370081Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.593713078Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883,"session_id":"17fb632b-7fb7-4dd8-ad0c-324dbc521c7a"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:06.593725617Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 44 len 12288 data: 13 12 15 02/30 Write block 56 len 32768 data: 11 14 15 14 14 11 13 16 03/30 Flush 04/30 Write block 94 len 24576 data: 11 13 11 14 17 20 05/30 Write block 416 len 24576 data: 18 20 18 22 18 18 06/30 Read block 170 len 20480 07/30 Read block 414 len 16384 08/30 Write block 380 len 28672 data: 17 13 12 8 9 7 8 09/30 Write block 281 len 36864 data: 6 8 13 10 12 12 12 12 13 10/30 Write block 315 len 16384 data: 13 14 17 20 11/30 Read block 498 len 16384 12/30 Write block 526 len 16384 data: 14 14 12 13 13/30 Read block 109 len 20480 14/30 Flush 15/30 Write block 551 len 36864 data: 18 18 16 14 14 11 10 11 9 16/30 Read block 90 len 28672 17/30 Read block 344 len 12288 18/30 Read block 366 len 40960 19/30 Flush 20/30 Write block 170 len 8192 data: 8 9 21/30 Write block 110 len 12288 data: 7 8 11 22/30 Write block 397 len 16384 data: 14 15 15 14 23/30 Write block 454 len 40960 data: 14 16 14 19 18 17 16 16 13 10 24/30 Read block 340 len 4096 25/30 Write block 500 len 24576 data: 16 16 12 13 12 12 26/30 Read block 388 len 32768 27/30 Read block 561 len 24576 28/30 Read block 580 len 16384 29/30 Read block 145 len 8192 30/30 Read block 235 len 4096 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 2 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 1 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 10 11 11 32 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:06.61294774Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1883} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 267 267 267 F F F 1 020-039 172 172 172 272 272 272 F F F 2 040-059 174 174 174 274 274 274 F F F 3 060-079 174 174 174 274 274 274 F F F 4 080-099 174 174 174 275 275 275 F F F 5 100-119 173 173 173 273 273 273 T T T 6 120-139 172 172 172 270 270 270 F F F 7 140-159 172 172 172 271 271 271 F F F 8 160-179 173 173 173 273 273 273 T T T 9 180-199 167 167 167 264 264 264 F F F 10 200-219 172 172 172 270 270 270 F F F 11 220-239 173 173 173 273 273 273 F F F 12 240-259 172 172 172 271 271 271 F F F 13 260-279 168 168 168 265 265 265 F F F 14 280-299 174 174 174 275 275 275 F F F 15 300-319 174 174 174 275 275 275 F F F 16 320-339 165 165 165 259 259 259 F F F 17 340-359 153 153 153 236 236 236 F F F 18 360-379 171 171 171 269 269 269 F F F 19 380-399 174 174 174 275 275 275 T T T 20 400-419 174 174 174 275 275 275 T T T 21 420-439 174 174 174 275 275 275 F F F 22 440-459 164 164 164 258 258 258 T T T 23 460-479 170 170 170 268 268 268 T T T 24 480-499 168 168 168 265 265 265 F F F 25 500-519 168 168 168 265 265 265 T T T 26 520-539 174 174 174 275 275 275 F F F 27 540-559 174 174 174 276 276 276 F F F 28 560-579 171 171 171 269 269 269 F F F 29 580-599 171 171 171 269 269 269 F F F Max gen: 174, Max flush: 276 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 267 267 267 F F F 1 020-039 172 172 172 272 272 272 F F F 2 040-059 174 174 174 274 274 274 F F F 3 060-079 174 174 174 274 274 274 F F F 4 080-099 174 174 174 275 275 275 F F F 5 100-119 175 175 175 277 277 277 F F F 6 120-139 172 172 172 270 270 270 F F F 7 140-159 172 172 172 271 271 271 F F F 8 160-179 175 175 175 277 277 277 F F F 9 180-199 167 167 167 264 264 264 F F F 10 200-219 172 172 172 270 270 270 F F F 11 220-239 173 173 173 273 273 273 F F F 12 240-259 172 172 172 271 271 271 F F F 13 260-279 168 168 168 265 265 265 F F F 14 280-299 174 174 174 275 275 275 F F F 15 300-319 174 174 174 275 275 275 F F F 16 320-339 165 165 165 259 259 259 F F F 17 340-359 153 153 153 236 236 236 F F F 18 360-379 171 171 171 269 269 269 F F F 19 380-399 175 175 175 277 277 277 F F F 20 400-419 175 175 175 277 277 277 F F F 21 420-439 174 174 174 275 275 275 F F F 22 440-459 175 175 175 277 277 277 F F F 23 460-479 175 175 175 277 277 277 F F F 24 480-499 168 168 168 265 265 265 F F F 25 500-519 175 175 175 277 277 277 F F F 26 520-539 174 174 174 275 275 275 F F F 27 540-559 174 174 174 276 276 276 F F F 28 560-579 171 171 171 269 269 269 F F F 29 580-599 171 171 171 269 269 269 F F F Max gen: 175, Max flush: 277 Begin loop 88 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:10.892128139Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:10.934621575Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:10.935002609Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:10.967082565Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:10.967639074Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:10.967683367Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:10.967701604Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:10.967720482Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891} {"msg":"Crucible 2ccc7a35-4fc6-44d7-a477-4d055b019bb7 has session id: eee5125c-1cad-4336-9ed5-22a80ffb293e","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:10.968187523Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"Upstairs opts: Upstairs UUID: 2ccc7a35-4fc6-44d7-a477-4d055b019bb7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:10.968249085Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"Crucible stats registered with UUID: 2ccc7a35-4fc6-44d7-a477-4d055b019bb7","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:10.968283089Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:10.999149602Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:10.999180558Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891} {"msg":"2ccc7a35-4fc6-44d7-a477-4d055b019bb7 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:10.999213013Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:10.999230451Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"client":"0","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:10.999257197Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"client":"0","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:10.999281783Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"client":"1","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:10.999314998Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"client":"1","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:10.999348813Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"client":"2","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:10.999380289Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"client":"2","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:10.999409524Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"":"io task","client":"2","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:10.99943929Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"":"io task","client":"0","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:10.999469536Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"":"io task","client":"1","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"ds_connection connected from Ok(127.0.0.1:61013)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:10.999503571Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"":"io task","client":"2","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"ds_connection connected from Ok(127.0.0.1:60609)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:10.999533626Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"":"io task","client":"0","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"ds_connection connected from Ok(127.0.0.1:45229)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:10.999563582Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"":"io task","client":"1","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.000179682Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"client":"2","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.000221726Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"client":"2","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.000255611Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"client":"1","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.000284597Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"client":"1","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.000462921Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"client":"0","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.000493096Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"client":"0","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.001025119Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"client":"1","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.001056064Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"client":"2","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.001073522Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"client":"0","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.001644118Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"client":"0","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.001672524Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"client":"0","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.001687282Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.00176721Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"client":"2","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.001792107Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"client":"2","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.001806614Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.00183381Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"client":"1","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.001848139Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"client":"1","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"[0]R flush_numbers[0..12]: [267, 272, 274, 274, 275, 277, 270, 271, 277, 264, 270, 273]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.001874395Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"":"downstairs","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"[0]R generation[0..12]: [170, 172, 174, 174, 174, 175, 172, 172, 175, 167, 172, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.001889262Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"":"downstairs","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.00190354Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"":"downstairs","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"[1]R flush_numbers[0..12]: [267, 272, 274, 274, 275, 277, 270, 271, 277, 264, 270, 273]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.001917778Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"":"downstairs","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"[1]R generation[0..12]: [170, 172, 174, 174, 174, 175, 172, 172, 175, 167, 172, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.001931646Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"":"downstairs","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.001945834Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"":"downstairs","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"[2]R flush_numbers[0..12]: [267, 272, 274, 274, 275, 277, 270, 271, 277, 264, 270, 273]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.001959682Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"":"downstairs","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"[2]R generation[0..12]: [170, 172, 174, 174, 174, 175, 172, 172, 175, 167, 172, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.00197358Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"":"downstairs","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.001987548Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"":"downstairs","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.002000936Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"":"downstairs","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.002017824Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"":"downstairs","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"Next flush: 278","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.002031612Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"":"downstairs","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.00204492Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"":"downstairs","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.002058358Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:37:11.002070786Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"client":"0","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.002090803Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"client":"1","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.002104321Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"client":"2","":"downstairs","session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.002117709Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.002130097Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"2ccc7a35-4fc6-44d7-a477-4d055b019bb7 is now active with session: eee5125c-1cad-4336-9ed5-22a80ffb293e","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.002143036Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.002155324Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891,"session_id":"eee5125c-1cad-4336-9ed5-22a80ffb293e"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:11.002167742Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 310 len 24576 data: 12 13 13 14 16 14 02/30 Write block 167 len 24576 data: 21 12 10 9 10 8 03/30 Write block 46 len 4096 data: 16 04/30 Write block 395 len 32768 data: 11 11 15 16 16 15 13 12 05/30 Write block 72 len 12288 data: 14 13 16 06/30 Flush 07/30 Write block 102 len 36864 data: 15 15 13 11 13 12 13 11 8 08/30 Write block 174 len 36864 data: 9 12 14 14 12 13 10 10 13 09/30 Read block 377 len 36864 10/30 Read block 310 len 36864 11/30 Flush 12/30 Write block 471 len 36864 data: 16 16 17 11 11 16 14 12 11 13/30 Flush 14/30 Write block 520 len 32768 data: 12 13 14 15 15 15 15 15 15/30 Read block 221 len 8192 16/30 Write block 159 len 12288 data: 14 17 19 17/30 Read block 506 len 20480 18/30 Write block 435 len 40960 data: 11 13 9 12 11 12 14 14 14 14 19/30 Read block 346 len 28672 20/30 Read block 243 len 32768 21/30 Write block 110 len 28672 data: 9 9 12 11 12 13 17 22/30 Read block 223 len 28672 23/30 Read block 426 len 36864 24/30 Write block 330 len 24576 data: 12 9 8 9 10 8 25/30 Write block 226 len 24576 data: 13 16 18 15 16 13 26/30 Write block 165 len 20480 data: 20 19 22 13 11 27/30 Read block 348 len 32768 28/30 Read block 227 len 8192 29/30 Write block 154 len 4096 data: 9 30/30 Write block 432 len 20480 data: 9 7 8 12 14 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 8 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:11.021348061Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1891} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 267 267 267 F F F 1 020-039 172 172 172 272 272 272 F F F 2 040-059 176 176 176 278 278 278 F F F 3 060-079 176 176 176 278 278 278 F F F 4 080-099 174 174 174 275 275 275 F F F 5 100-119 176 176 176 279 279 279 T T T 6 120-139 172 172 172 270 270 270 F F F 7 140-159 172 172 172 271 271 271 T T T 8 160-179 176 176 176 279 279 279 T T T 9 180-199 176 176 176 279 279 279 F F F 10 200-219 172 172 172 270 270 270 F F F 11 220-239 173 173 173 273 273 273 T T T 12 240-259 172 172 172 271 271 271 F F F 13 260-279 168 168 168 265 265 265 F F F 14 280-299 174 174 174 275 275 275 F F F 15 300-319 176 176 176 278 278 278 F F F 16 320-339 165 165 165 259 259 259 T T T 17 340-359 153 153 153 236 236 236 F F F 18 360-379 171 171 171 269 269 269 F F F 19 380-399 176 176 176 278 278 278 F F F 20 400-419 176 176 176 278 278 278 F F F 21 420-439 174 174 174 275 275 275 T T T 22 440-459 175 175 175 277 277 277 T T T 23 460-479 176 176 176 280 280 280 F F F 24 480-499 168 168 168 265 265 265 F F F 25 500-519 175 175 175 277 277 277 F F F 26 520-539 174 174 174 275 275 275 T T T 27 540-559 174 174 174 276 276 276 F F F 28 560-579 171 171 171 269 269 269 F F F 29 580-599 171 171 171 269 269 269 F F F Max gen: 176, Max flush: 280 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 267 267 267 F F F 1 020-039 172 172 172 272 272 272 F F F 2 040-059 176 176 176 278 278 278 F F F 3 060-079 176 176 176 278 278 278 F F F 4 080-099 174 174 174 275 275 275 F F F 5 100-119 177 177 177 281 281 281 F F F 6 120-139 172 172 172 270 270 270 F F F 7 140-159 177 177 177 281 281 281 F F F 8 160-179 177 177 177 281 281 281 F F F 9 180-199 176 176 176 279 279 279 F F F 10 200-219 172 172 172 270 270 270 F F F 11 220-239 177 177 177 281 281 281 F F F 12 240-259 172 172 172 271 271 271 F F F 13 260-279 168 168 168 265 265 265 F F F 14 280-299 174 174 174 275 275 275 F F F 15 300-319 176 176 176 278 278 278 F F F 16 320-339 177 177 177 281 281 281 F F F 17 340-359 153 153 153 236 236 236 F F F 18 360-379 171 171 171 269 269 269 F F F 19 380-399 176 176 176 278 278 278 F F F 20 400-419 176 176 176 278 278 278 F F F 21 420-439 177 177 177 281 281 281 F F F 22 440-459 177 177 177 281 281 281 F F F 23 460-479 176 176 176 280 280 280 F F F 24 480-499 168 168 168 265 265 265 F F F 25 500-519 175 175 175 277 277 277 F F F 26 520-539 177 177 177 281 281 281 F F F 27 540-559 174 174 174 276 276 276 F F F 28 560-579 171 171 171 269 269 269 F F F 29 580-599 171 171 171 269 269 269 F F F Max gen: 177, Max flush: 281 Begin loop 89 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.338519794Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.382732898Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.383345289Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.383381703Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.383396481Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.383410949Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899} {"msg":"Crucible 9321a055-d53c-471d-90ae-1573f5d8021b has session id: 13dfdd53-adc5-4ba8-9471-507400c382e3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.383912106Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"Upstairs opts: Upstairs UUID: 9321a055-d53c-471d-90ae-1573f5d8021b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.3839552Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"Crucible stats registered with UUID: 9321a055-d53c-471d-90ae-1573f5d8021b","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.383972717Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.414828202Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.414862577Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899} {"msg":"9321a055-d53c-471d-90ae-1573f5d8021b active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.414887543Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.414904681Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"client":"0","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.414932367Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"client":"0","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.414950954Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"client":"1","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.414968631Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"client":"1","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.414985819Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"client":"2","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.415003136Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"client":"2","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.415020334Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"":"io task","client":"2","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.41505033Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"":"io task","client":"0","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.415085474Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"":"io task","client":"1","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"ds_connection connected from Ok(127.0.0.1:45906)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.415130188Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"":"io task","client":"1","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"ds_connection connected from Ok(127.0.0.1:54193)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.415172882Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"":"io task","client":"0","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"ds_connection connected from Ok(127.0.0.1:63658)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.415219025Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"":"io task","client":"2","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.415928531Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"client":"2","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.415956727Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"client":"2","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.415975864Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"client":"0","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.41600352Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"client":"0","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.416122723Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"client":"1","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.416152079Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"client":"1","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.416569718Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"client":"0","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.416601763Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"client":"2","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.416705398Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"client":"1","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.417182678Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"client":"2","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.417210384Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"client":"2","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.417225092Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.417242219Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"client":"0","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.417275534Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"client":"0","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.417291092Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.41730781Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"client":"1","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.417321518Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"client":"1","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"[0]R flush_numbers[0..12]: [267, 272, 278, 278, 275, 281, 270, 281, 281, 279, 270, 281]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.417350514Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"":"downstairs","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"[0]R generation[0..12]: [170, 172, 176, 176, 174, 177, 172, 177, 177, 176, 172, 177]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.417365352Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"":"downstairs","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.417379769Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"":"downstairs","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"[1]R flush_numbers[0..12]: [267, 272, 278, 278, 275, 281, 270, 281, 281, 279, 270, 281]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.417394007Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"":"downstairs","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"[1]R generation[0..12]: [170, 172, 176, 176, 174, 177, 172, 177, 177, 176, 172, 177]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.417407815Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"":"downstairs","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.417421923Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"":"downstairs","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"[2]R flush_numbers[0..12]: [267, 272, 278, 278, 275, 281, 270, 281, 281, 279, 270, 281]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.417435851Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"":"downstairs","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"[2]R generation[0..12]: [170, 172, 176, 176, 174, 177, 172, 177, 177, 176, 172, 177]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.417449759Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"":"downstairs","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.417463867Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"":"downstairs","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.417476955Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"":"downstairs","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.417492373Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"":"downstairs","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"Next flush: 282","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.417505881Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"":"downstairs","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.417519149Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"":"downstairs","":"downstairs","session_id"Wait for a query_work_queue command to finish before sending IO :"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.417541316Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.417553854Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"client":"0","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.417567442Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"client":"1","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.41758075Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"client":"2","":"downstairs","session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.417594258Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.417606706Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"9321a055-d53c-471d-90ae-1573f5d8021b is now active with session: 13dfdd53-adc5-4ba8-9471-507400c382e3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.417619734Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.417631842Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899,"session_id":"13dfdd53-adc5-4ba8-9471-507400c382e3"} {"msg":"The guest has finished waiting for activation with:178","v":0Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"name":"crucible","level":30,"time":"2025-01-03T08:37:15.41764774Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 227 len 28672 data: 17 19 16 17 14 13 16 03/30 Flush 04/30 Write block 147 len 20480 data: 18 14 15 10 7 05/30 Write block 538 len 24576 data: 8 8 9 11 14 15 06/30 Read block 345 len 12288 07/30 Read block 145 len 36864 08/30 Read block 273 len 12288 09/30 Read block 234 len 20480 10/30 Write block 361 len 40960 data: 12 10 10 13 16 14 14 15 14 17 11/30 Write block 452 len 28672 data: 12 12 15 17 15 20 19 12/30 Read block 379 len 24576 13/30 Read block 406 len 40960 14/30 Write block 169 len 20480 data: 12 10 11 9 6 15/30 Write block 549 len 12288 data: 17 17 19 16/30 Read block 524 len 24576 17/30 Write block 207 len 16384 data: 9 10 10 10 18/30 Read block 82 len 28672 19/30 Write block 412 len 12288 data: 23 21 24 20/30 Write block 128 len 16384 data: 14 18 16 17 21/30 Read block 415 len 12288 22/30 Read block 35 len 32768 23/30 Read block 534 len 8192 24/30 Write block 74 len 8192 data: 17 17 25/30 Write block 507 len 28672 data: 12 14 12 12 11 9 11 26/30 Read block 291 len 36864 27/30 Read block 382 len 24576 28/30 Read block 100 len 12288 29/30 Read block 449 len 4096 30/30 Read block 464 len 32768 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 5 Done Done Done false 1004 Acked Write 6 Done Done Done false 1005 Acked Read 3 Done Done Done false 1006 Acked Read 9 Done Done Done false 1007 Acked Read 3 Done Done Done false 1008 Acked Read 5 Done Done Done false 1009 Acked Write 10 Done Done Done false 1010 Acked Write 7 Done Done Done false 1011 Acked Read 6 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Write 5 Done Done Done false 1014 Acked Write 3 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 26 26 27 79 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1002 1002 1002 Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:15.437253257Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1899} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 267 267 267 F F F 1 020-039 172 172 172 272 272 272 F F F 2 040-059 176 176 176 278 278 278 F F F 3 060-079 176 176 176 278 278 278 T T T 4 080-099 174 174 174 275 275 275 F F F 5 100-119 177 177 177 281 281 281 F F F 6 120-139 172 172 172 270 270 270 T T T 7 140-159 177 177 177 281 281 281 T T T 8 160-179 177 177 177 281 281 281 T T T 9 180-199 176 176 176 279 279 279 F F F 10 200-219 172 172 172 270 270 270 T T T 11 220-239 178 178 178 283 283 283 F F F 12 240-259 172 172 172 271 271 271 F F F 13 260-279 168 168 168 265 265 265 F F F 14 280-299 174 174 174 275 275 275 F F F 15 300-319 176 176 176 278 278 278 F F F 16 320-339 177 177 177 281 281 281 F F F 17 340-359 153 153 153 236 236 236 F F F 18 360-379 171 171 171 269 269 269 T T T 19 380-399 176 176 176 278 278 278 F F F 20 400-419 176 176 176 278 278 278 T T T 21 420-439 177 177 177 281 281 281 F F F 22 440-459 177 177 177 281 281 281 T T T 23 460-479 176 176 176 280 280 280 F F F 24 480-499 168 168 168 265 265 265 F F F 25 500-519 175 175 175 277 277 277 T T T 26 520-539 177 177 177 281 281 281 T T T 27 540-559 174 174 174 276 276 276 T T T 28 560-579 171 171 171 269 269 269 F F F 29 580-599 171 171 171 269 269 269 F F F Max gen: 178, Max flush: 283 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 267 267 267 F F F 1 020-039 172 172 172 272 272 272 F F F 2 040-059 176 176 176 278 278 278 F F F 3 060-079 179 179 179 284 284 284 F F F 4 080-099 174 174 174 275 275 275 F F F 5 100-119 177 177 177 281 281 281 F F F 6 120-139 179 179 179 284 284 284 F F F 7 140-159 179 179 179 284 284 284 F F F 8 160-179 179 179 179 284 284 284 F F F 9 180-199 176 176 176 279 279 279 F F F 10 200-219 179 179 179 284 284 284 F F F 11 220-239 178 178 178 283 283 283 F F F 12 240-259 172 172 172 271 271 271 F F F 13 260-279 168 168 168 265 265 265 F F F 14 280-299 174 174 174 275 275 275 F F F 15 300-319 176 176 176 278 278 278 F F F 16 320-339 177 177 177 281 281 281 F F F 17 340-359 153 153 153 236 236 236 F F F 18 360-379 179 179 179 284 284 284 F F F 19 380-399 176 176 176 278 278 278 F F F 20 400-419 179 179 179 284 284 284 F F F 21 420-439 177 177 177 281 281 281 F F F 22 440-459 179 179 179 284 284 284 F F F 23 460-479 176 176 176 280 280 280 F F F 24 480-499 168 168 168 265 265 265 F F F 25 500-519 179 179 179 284 284 284 F F F 26 520-539 179 179 179 284 284 284 F F F 27 540-559 179 179 179 284 284 284 F F F 28 560-579 171 171 171 269 269 269 F F F 29 580-599 171 171 171 269 269 269 F F F Max gen: 179, Max flush: 284 Begin loop 90 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.740821017Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:19.782701871Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.783067237Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.814238436Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.814830329Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.814862705Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.814876513Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.81489553Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907} {"msg":"Crucible b40b290f-d36f-4c68-8ad0-64459decaa56 has session id: 3031277c-72b0-431f-86d5-8662a30bbd65","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.815361942Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"Upstairs opts: Upstairs UUID: b40b290f-d36f-4c68-8ad0-64459decaa56, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.815395477Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"Crucible stats registered with UUID: b40b290f-d36f-4c68-8ad0-64459decaa56","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.815410405Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.846113012Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.846143117Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907} {"msg":"b40b290f-d36f-4c68-8ad0-64459decaa56 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.846214237Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.846243653Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"client":"0","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.846284407Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"client":"0","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.846311563Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"client":"1","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.846337679Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"client":"1","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.846363035Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"client":"2","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.846404619Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"client":"2","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.846451942Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"":"io task","client":"2","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.846497965Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"":"io task","client":"0","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.846556187Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"":"io task","client":"1","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"ds_connection connected from Ok(127.0.0.1:50892)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.846630566Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"":"io task","client":"2","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"ds_connection connected from Ok(127.0.0.1:54876)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.846683178Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"":"io task","client":"1","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"ds_connection connected from Ok(127.0.0.1:47816)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.846897107Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"":"io task","client":"0","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.847390155Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"client":"1","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.847419891Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"client":"1","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.847547352Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"client":"2","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.847573159Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"client":"2","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.847778229Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"client":"0","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.847799366Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"client":"0","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.847923957Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"client":"1","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.848236002Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"client":"2","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.848376551Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"client":"0","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.848535538Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"client":"1","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.848557865Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"client":"1","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.848577212Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.848717301Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"client":"2","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.848738368Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"client":"2","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.848752496Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.848871739Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"client":"0","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.848891436Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"client":"0","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"[0]R flush_numbers[0..12]: [267, 272, 278, 284, 275, 281, 284, 284, 284, 279, 284, 283]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.848905914Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"":"downstairs","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"[0]R generation[0..12]: [170, 172, 176, 179, 174, 177, 179, 179, 179, 176, 179, 178]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.848919922Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"":"downstairs","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.84893407Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"":"downstairs","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"[1]R flush_numbers[0..12]: [267, 272, 278, 284, 275, 281, 284, 284, 284, 279, 284, 283]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.848948438Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"":"downstairs","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"[1]R generation[0..12]: [170, 172, 176, 179, 174, 177, 179, 179, 179, 176, 179, 178]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.848962506Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"":"downstairs","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.848976604Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"":"downstairs","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"[2]R flush_numbers[0..12]: [267, 272, 278, 284, 275, 281, 284, 284, 284, 279, 284, 283]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.848990521Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"":"downstairs","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"[2]R generation[0..12]: [170, 172, 176, 179, 174, 177, 179, 179, 179, 176, 179, 178]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.84900433Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"":"downstairs","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.849021817Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"":"downstairs","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.849035435Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"":"downstairs","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.849048823Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"":"downstairs","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"Next flush: 285","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.849061811Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"":"downstairs","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:37:19.849074819Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"":"downstairs","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.849096196Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.849108394Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"client":"0","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.849121932Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"client":"1","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.849135191Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"client":"2","":"downstairs","session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.849149318Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.849161816Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"b40b290f-d36f-4c68-8ad0-64459decaa56 is now active with session: 3031277c-72b0-431f-86d5-8662a30bbd65","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.849174675Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.849186853Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907,"session_id":"3031277c-72b0-431f-86d5-8662a30bbd65"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:19.849199201Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 232 len 28672 data: 14 17 15 15 16 16 15 02/30 Read block 558 len 20480 03/30 Read block 407 len 20480 04/30 Read block 551 len 4096 05/30 Write block 282 len 12288 data: 9 14 11 06/30 Flush 07/30 Write block 523 len 4096 data: 16 08/30 Write block 336 len 32768 data: 9 9 10 9 8 10 11 9 09/30 Write block 453 len 4096 data: 13 10/30 Write block 399 len 20480 data: 17 16 14 13 10 11/30 Read block 340 len 24576 12/30 Write block 196 len 4096 data: 14 13/30 Read block 32 len 32768 14/30 Write block 297 len 24576 data: 11 8 10 11 10 12 15/30 Write block 376 len 12288 data: 20 21 18 16/30 Write block 369 len 32768 data: 15 18 18 19 19 18 19 21 17/30 Write block 528 len 8192 data: 13 14 18/30 Write block 124 len 4096 data: 18 19/30 Read block 286 len 8192 20/30 Write block 131 len 24576 data: 18 15 13 11 11 8 21/30 Flush 22/30 Write block 106 len 28672 data: 14 13 14 12 10 10 13 23/30 Write block 334 len 32768 data: 11 9 10 10 11 10 9 11 24/30 Write block 48 len 32768 data: 11 12 14 14 16 14 14 11 25/30 Read block 418 len 32768 26/30 Write block 480 len 20480 data: 13 13 13 14 13 27/30 Read block 290 len 24576 28/30 Read block 104 len 20480 29/30 Read block 63 len 8192 30/30 Read block 161 len 4096 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 7 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 9 8 8 25 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:19.866478458Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1907} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 267 267 267 F F F 1 020-039 172 172 172 272 272 272 F F F 2 040-059 176 176 176 278 278 278 T T T 3 060-079 179 179 179 284 284 284 F F F 4 080-099 174 174 174 275 275 275 F F F 5 100-119 177 177 177 281 281 281 T T T 6 120-139 180 180 180 286 286 286 F F F 7 140-159 179 179 179 284 284 284 F F F 8 160-179 179 179 179 284 284 284 F F F 9 180-199 180 180 180 286 286 286 F F F 10 200-219 179 179 179 284 284 284 F F F 11 220-239 180 180 180 285 285 285 F F F 12 240-259 172 172 172 271 271 271 F F F 13 260-279 168 168 168 265 265 265 F F F 14 280-299 180 180 180 286 286 286 F F F 15 300-319 180 180 180 286 286 286 F F F 16 320-339 180 180 180 286 286 286 T T T 17 340-359 180 180 180 286 286 286 T T T 18 360-379 180 180 180 286 286 286 F F F 19 380-399 180 180 180 286 286 286 F F F 20 400-419 180 180 180 286 286 286 F F F 21 420-439 177 177 177 281 281 281 F F F 22 440-459 180 180 180 286 286 286 F F F 23 460-479 176 176 176 280 280 280 F F F 24 480-499 168 168 168 265 265 265 T T T 25 500-519 179 179 179 284 284 284 F F F 26 520-539 180 180 180 286 286 286 F F F 27 540-559 179 179 179 284 284 284 F F F 28 560-579 171 171 171 269 269 269 F F F 29 580-599 171 171 171 269 269 269 F F F Max gen: 180, Max flush: 286 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 267 267 267 F F F 1 020-039 172 172 172 272 272 272 F F F 2 040-059 181 181 181 287 287 287 F F F 3 060-079 179 179 179 284 284 284 F F F 4 080-099 174 174 174 275 275 275 F F F 5 100-119 181 181 181 287 287 287 F F F 6 120-139 180 180 180 286 286 286 F F F 7 140-159 179 179 179 284 284 284 F F F 8 160-179 179 179 179 284 284 284 F F F 9 180-199 180 180 180 286 286 286 F F F 10 200-219 179 179 179 284 284 284 F F F 11 220-239 180 180 180 285 285 285 F F F 12 240-259 172 172 172 271 271 271 F F F 13 260-279 168 168 168 265 265 265 F F F 14 280-299 180 180 180 286 286 286 F F F 15 300-319 180 180 180 286 286 286 F F F 16 320-339 181 181 181 287 287 287 F F F 17 340-359 181 181 181 287 287 287 F F F 18 360-379 180 180 180 286 286 286 F F F 19 380-399 180 180 180 286 286 286 F F F 20 400-419 180 180 180 286 286 286 F F F 21 420-439 177 177 177 281 281 281 F F F 22 440-459 180 180 180 286 286 286 F F F 23 460-479 176 176 176 280 280 280 F F F 24 480-499 181 181 181 287 287 287 F F F 25 500-519 179 179 179 284 284 284 F F F 26 520-539 180 180 180 286 286 286 F F F 27 540-559 179 179 179 284 284 284 F F F 28 560-579 171 171 171 269 269 269 F F F 29 580-599 171 171 171 269 269 269 F F F Max gen: 181, Max flush: 287 Begin loop 91 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.16453887Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.207742382Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.208429692Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.208467336Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.208482104Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.208497362Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915} {"msg":"Crucible 5eb9777c-7e01-4acd-b01b-4afff5c966ec has session id: e92d57e1-9011-4495-95e9-bb89f88e0214","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.208950396Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"Upstairs opts: Upstairs UUID: 5eb9777c-7e01-4acd-b01b-4afff5c966ec, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.209001938Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"Crucible stats registered with UUID: 5eb9777c-7e01-4acd-b01b-4afff5c966ec","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.209028984Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.239411418Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.239444663Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915} {"msg":"5eb9777c-7e01-4acd-b01b-4afff5c966ec active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.239478418Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.239495716Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"client":"0","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.239522302Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"client":"0","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.239540139Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"client":"1","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.239557077Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"client":"1","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.239578964Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"client":"2","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.239611699Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"client":"2","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.239643914Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"":"io task","client":"2","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.239677299Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"":"io task","client":"0","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.239711554Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"":"io task","client":"1","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"ds_connection connected from Ok(127.0.0.1:36670)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.239748029Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"":"io task","client":"2","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"ds_connection connected from Ok(127.0.0.1:56307)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.239787323Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"":"io task","client":"1","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"ds_connection connected from Ok(127.0.0.1:60000)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.239953519Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"":"io task","client":"0","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.240611273Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"client":"0","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.240640509Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"client":"0","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.240655156Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"client":"2","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.240670884Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"client":"2","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.240745773Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"client":"1","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.240786767Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"client":"1","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.241305552Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"client":"0","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.241334877Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"client":"1","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.241354085Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"client":"2","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.241873169Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"client":"0","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.241902245Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"client":"0","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.241917432Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.241985042Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"client":"1","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.242008599Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"client":"1","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.242023567Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.242036145Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"client":"2","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.242049613Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"client":"2","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"[0]R flush_numbers[0..12]: [267, 272, 287, 284, 275, 287, 286, 284, 284, 286, 284, 285]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.242063511Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"":"downstairs","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"[0]R generation[0..12]: [170, 172, 181, 179, 174, 181, 180, 179, 179, 180, 179, 180]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.242077549Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"":"downstairs","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.242091877Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"":"downstairs","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"[1]R flush_numbers[0..12]: [267, 272, 287, 284, 275, 287, 286, 284, 284, 286, 284, 285]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.242105635Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"":"downstairs","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"[1]R generation[0..12]: [170, 172, 181, 179, 174, 181, 180, 179, 179, 180, 179, 180]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.242119443Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"":"downstairs","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.242133611Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"":"downstairs","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"[2]R flush_numbers[0..12]: [267, 272, 287, 284, 275, 287, 286, 284, 284, 286, 284, 285]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.242147159Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"":"downstairs","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"[2]R generation[0..12]: [170, 172, 181, 179, 174, 181, 180, 179, 179, 180, 179, 180]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.242160937Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"":"downstairs","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.242174875Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"":"downstairs","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.242187983Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"":"downstairs","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.242203461Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"":"downstairsWait for a query_work_queue command to finish before sending IO ","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"Next flush: 288","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.242284399Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"":"downstairs","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.242300436Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"":"downstairs","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.242314054Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.242326292Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"client":"0","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.242340021Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"client":"1","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.242353448Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"client":"2","":"downstairs","session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.242366966Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.242379345Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"5eb9777c-7e01-4acd-b01b-4afff5c966ec is now active with session: e92d57e1-9011-4495-95e9-bb89f88e0214","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.242392053Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.242404141Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915,"session_id":"e92d57e1-9011-4495-95e9-bb89f88e0214"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:24.242416349Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 105 len 12288 data: 12 15 14 02/30 Read block 396 len 32768 03/30 Write block 227 len 28672 data: 18 20 17 18 15 15 18 04/30 Write block 133 len 36864 data: 14 12 12 9 8 7 9 9 11 05/30 Read block 63 len 12288 06/30 Flush 07/30 Write block 78 len 4096 data: 14 08/30 Read block 70 len 36864 09/30 Flush 10/30 Write block 156 len 36864 data: 14 13 15 15 18 20 21 21 22 11/30 Flush 12/30 Write block 99 len 36864 data: 21 17 16 16 16 14 13 16 15 13/30 Read block 96 len 24576 14/30 Read block 525 len 12288 15/30 Write block 223 len 36864 data: 14 15 12 14 19 21 18 19 16 16/30 Write block 454 len 8192 data: 16 18 17/30 Flush 18/30 Write block 462 len 4096 data: 14 19/30 Write block 409 len 8192 data: 17 19 20/30 Read block 145 len 20480 21/30 Read block 328 len 28672 22/30 Write block 572 len 24576 data: 12 12 6 8 9 13 23/30 Flush 24/30 Write block 146 len 20480 data: 17 19 15 16 11 25/30 Read block 212 len 28672 26/30 Read block 578 len 8192 27/30 Write block 436 len 40960 data: 15 10 13 12 13 15 15 15 15 15 28/30 Read block 85 len 40960 29/30 Write block 548 len 20480 data: 21 18 18 20 19 30/30 Read block 468 len 24576 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 5 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 5 Done Sent Done false 1029 Acked Read 6 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 0 3 Done 6 5 7 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:24.262360907Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1915} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 267 267 267 F F F 1 020-039 172 172 172 272 272 272 F F F 2 040-059 181 181 181 287 287 287 F F F 3 060-079 182 182 182 289 289 289 F F F 4 080-099 182 182 182 291 291 291 F F F 5 100-119 182 182 182 291 291 291 F F F 6 120-139 182 182 182 288 288 288 F F F 7 140-159 182 182 182 290 290 290 T T T 8 160-179 182 182 182 290 290 290 F F F 9 180-199 180 180 180 286 286 286 F F F 10 200-219 179 179 179 284 284 284 F F F 11 220-239 182 182 182 291 291 291 F F F 12 240-259 172 172 172 271 271 271 F F F 13 260-279 168 168 168 265 265 265 F F F 14 280-299 180 180 180 286 286 286 F F F 15 300-319 180 180 180 286 286 286 F F F 16 320-339 181 181 181 287 287 287 F F F 17 340-359 181 181 181 287 287 287 F F F 18 360-379 180 180 180 286 286 286 F F F 19 380-399 180 180 180 286 286 286 F F F 20 400-419 182 182 182 292 292 292 F F F 21 420-439 177 177 177 281 281 281 T T T 22 440-459 182 182 182 291 291 291 T T T 23 460-479 182 182 182 292 292 292 F F F 24 480-499 181 181 181 287 287 287 F F F 25 500-519 179 179 179 284 284 284 F F F 26 520-539 180 180 180 286 286 286 F F F 27 540-559 179 179 179 284 284 284 T T T 28 560-579 182 182 182 292 292 292 F F F 29 580-599 171 171 171 269 269 269 F F F Max gen: 182, Max flush: 292 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 267 267 267 F F F 1 020-039 172 172 172 272 272 272 F F F 2 040-059 181 181 181 287 287 287 F F F 3 060-079 182 182 182 289 289 289 F F F 4 080-099 182 182 182 291 291 291 F F F 5 100-119 182 182 182 291 291 291 F F F 6 120-139 182 182 182 288 288 288 F F F 7 140-159 183 183 183 293 293 293 F F F 8 160-179 182 182 182 290 290 290 F F F 9 180-199 180 180 180 286 286 286 F F F 10 200-219 179 179 179 284 284 284 F F F 11 220-239 182 182 182 291 291 291 F F F 12 240-259 172 172 172 271 271 271 F F F 13 260-279 168 168 168 265 265 265 F F F 14 280-299 180 180 180 286 286 286 F F F 15 300-319 180 180 180 286 286 286 F F F 16 320-339 181 181 181 287 287 287 F F F 17 340-359 181 181 181 287 287 287 F F F 18 360-379 180 180 180 286 286 286 F F F 19 380-399 180 180 180 286 286 286 F F F 20 400-419 182 182 182 292 292 292 F F F 21 420-439 183 183 183 293 293 293 F F F 22 440-459 183 183 183 293 293 293 F F F 23 460-479 182 182 182 292 292 292 F F F 24 480-499 181 181 181 287 287 287 F F F 25 500-519 179 179 179 284 284 284 F F F 26 520-539 180 180 180 286 286 286 F F F 27 540-559 183 183 183 293 293 293 F F F 28 560-579 182 182 182 292 292 292 F F F 29 580-599 171 171 171 269 269 269 F F F Max gen: 183, Max flush: 293 Begin loop 92 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.5254231Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.57281783Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.573605585Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.573650208Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.573668266Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.573686053Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923} {"msg":"Crucible 49b615af-398f-4d0e-a17f-f2ee17a17ea5 has session id: 7f2d2981-4c29-443e-bc36-f259485fde34","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.574235083Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"Upstairs opts: Upstairs UUID: 49b615af-398f-4d0e-a17f-f2ee17a17ea5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.574276077Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"Crucible stats registered with UUID: 49b615af-398f-4d0e-a17f-f2ee17a17ea5","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.574294064Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.604573723Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.604624656Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923} {"msg":"49b615af-398f-4d0e-a17f-f2ee17a17ea5 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.604657191Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.604687826Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"client":"0","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.60473589Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"client":"0","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.604773824Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"client":"1","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.604808869Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"client":"1","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.604841634Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"client":"2","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.604875409Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"client":"2","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.604909124Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"":"io task","client":"2","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.60494138Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"":"io task","client":"0","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.604975115Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"":"io task","client":"1","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"ds_connection connected from Ok(127.0.0.1:55779)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.60500212Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"":"io task","client":"2","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"ds_connection connected from Ok(127.0.0.1:41883)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.605025957Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"":"io task","client":"1","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"ds_connection connected from Ok(127.0.0.1:33574)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.605239966Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"":"io task","client":"0","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.605771578Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"client":"1","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.605799784Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"client":"1","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.605818711Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"client":"0","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.605833639Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"client":"0","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.605913438Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"client":"2","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.605932475Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"client":"2","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.606391248Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"client":"0","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.606485474Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"client":"1","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.606507871Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"client":"2","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.606953226Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"client":"0","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.606983301Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"client":"0","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.607004448Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.607085956Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"client":"1","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.607114962Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"client":"1","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.60713009Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.607142828Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"client":"2","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.607156856Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"client":"2","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"[0]R flush_numbers[0..12]: [267, 272, 287, 289, 291, 291, 288, 293, 290, 286, 284, 291]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.607174973Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"":"downstairs","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"[0]R generation[0..12]: [170, 172, 181, 182, 182, 182, 182, 183, 182, 180, 179, 182]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.607189501Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"":"downstairs","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.607203769Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"":"downstairs","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"[1]R flush_numbers[0..12]: [267, 272, 287, 289, 291, 291, 288, 293, 290, 286, 284, 291]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.607217947Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"":"downstairs","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"[1]R generation[0..12]: [170, 172, 181, 182, 182, 182, 182, 183, 182, 180, 179, 182]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.607231675Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"":"downstairs","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.607245553Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"":"downstairs","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"[2]R flush_numbers[0..12]: [267, 272, 287, 289, 291, 291, 288, 293, 290, 286, 284, 291]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.607259411Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"":"downstairs","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"[2]R generation[0..12]: [170, 172, 181, 182, 182, 182, 182, 183, 182, 180, 179, 182]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.607273309Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"":"downstairs","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.607287387Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"":"downstairs","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.607300545Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"":"downstairs","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.607313773Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"":"downstairs","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"Next flush: 294","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.60733479Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"":"downstairs","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.607347968Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"":"downstairs","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.607361706Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.607376534Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"client":"0","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.607390462Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"client":"1","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.60740395Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"client":"2","":"downstairs","session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.607417828Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.607430176Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"49b615af-398f-4d0e-a17f-f2ee17a17ea5 is now active with session: 7f2d2981-4c29-443e-bc36-f259485fde34","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.607442994Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.607455162Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923,"session_id":"7f2d2981-4c29-443e-bc36-f259485fde34"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:28.60746758Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 487 len 20480 data: 14 12 14 12 12 02/30 Write block 508 len 20480 data: 15 13 13 12 10 03/30 Write block 270 len 8192 data: 11 13 04/30 Read block 235 len 8192 05/30 Read block 478 len 8192 06/30 Write block 166 len 8192 data: 20 23 07/30 Read block 107 len 16384 08/30 Write block 483 len 28672 data: 15 14 13 11 15 13 15 09/30 Read block 439 len 36864 10/30 Read block 156 len 4096 11/30 Write block 228 len 8192 data: 22 19 12/30 Write block 65 len 32768 data: 16 16 17 17 14 15 13 15 13/30 Write block 417 len 28672 data: 21 19 23 19 19 15 11 14/30 Flush 15/30 Write block 536 len 36864 data: 8 9 9 9 10 12 15 16 15 16/30 Write block 338 len 32768 data: 12 11 10 12 12 10 10 12 17/30 Write block 10 len 32768 data: 10 9 11 11 12 14 14 14 18/30 Write block 230 len 28672 data: 20 17 16 19 16 16 17 19/30 Read block 176 len 16384 20/30 Write block 391 len 32768 data: 13 14 12 13 12 12 16 17 21/30 Flush 22/30 Write block 397 len 20480 data: 17 18 18 17 15 23/30 Write block 378 len 24576 data: 19 17 18 14 13 9 24/30 Read block 370 len 24576 25/30 Read block 207 len 16384 26/30 Write block 178 len 28672 data: 13 14 11 11 14 17 20 27/30 Write block 555 len 16384 data: 15 12 11 12 28/30 Read block 188 len 28672 29/30 Read block 570 len 32768 30/30 Read block 448 len 8192 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 5 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:28.625871723Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1923} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 295 295 295 F F F 1 020-039 172 172 172 272 272 272 F F F 2 040-059 181 181 181 287 287 287 F F F 3 060-079 184 184 184 294 294 294 F F F 4 080-099 182 182 182 291 291 291 F F F 5 100-119 182 182 182 291 291 291 F F F 6 120-139 182 182 182 288 288 288 F F F 7 140-159 183 183 183 293 293 293 F F F 8 160-179 184 184 184 294 294 294 T T T 9 180-199 180 180 180 286 286 286 T T T 10 200-219 179 179 179 284 284 284 F F F 11 220-239 184 184 184 295 295 295 F F F 12 240-259 172 172 172 271 271 271 F F F 13 260-279 184 184 184 294 294 294 F F F 14 280-299 180 180 180 286 286 286 F F F 15 300-319 180 180 180 286 286 286 F F F 16 320-339 184 184 184 295 295 295 F F F 17 340-359 184 184 184 295 295 295 F F F 18 360-379 180 180 180 286 286 286 T T T 19 380-399 184 184 184 295 295 295 T T T 20 400-419 184 184 184 294 294 294 T T T 21 420-439 184 184 184 294 294 294 F F F 22 440-459 183 183 183 293 293 293 F F F 23 460-479 182 182 182 292 292 292 F F F 24 480-499 184 184 184 294 294 294 F F F 25 500-519 184 184 184 294 294 294 F F F 26 520-539 184 184 184 295 295 295 F F F 27 540-559 184 184 184 295 295 295 T T T 28 560-579 182 182 182 292 292 292 F F F 29 580-599 171 171 171 269 269 269 F F F Max gen: 184, Max flush: 295 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 295 295 295 F F F 1 020-039 172 172 172 272 272 272 F F F 2 040-059 181 181 181 287 287 287 F F F 3 060-079 184 184 184 294 294 294 F F F 4 080-099 182 182 182 291 291 291 F F F 5 100-119 182 182 182 291 291 291 F F F 6 120-139 182 182 182 288 288 288 F F F 7 140-159 183 183 183 293 293 293 F F F 8 160-179 185 185 185 296 296 296 F F F 9 180-199 185 185 185 296 296 296 F F F 10 200-219 179 179 179 284 284 284 F F F 11 220-239 184 184 184 295 295 295 F F F 12 240-259 172 172 172 271 271 271 F F F 13 260-279 184 184 184 294 294 294 F F F 14 280-299 180 180 180 286 286 286 F F F 15 300-319 180 180 180 286 286 286 F F F 16 320-339 184 184 184 295 295 295 F F F 17 340-359 184 184 184 295 295 295 F F F 18 360-379 185 185 185 296 296 296 F F F 19 380-399 185 185 185 296 296 296 F F F 20 400-419 185 185 185 296 296 296 F F F 21 420-439 184 184 184 294 294 294 F F F 22 440-459 183 183 183 293 293 293 F F F 23 460-479 182 182 182 292 292 292 F F F 24 480-499 184 184 184 294 294 294 F F F 25 500-519 184 184 184 294 294 294 F F F 26 520-539 184 184 184 295 295 295 F F F 27 540-559 185 185 185 296 296 296 F F F 28 560-579 182 182 182 292 292 292 F F F 29 580-599 171 171 171 269 269 269 F F F Max gen: 185, Max flush: 296 Begin loop 93 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.899423705Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.944269037Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.94506882Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.945128501Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.945147709Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.945166786Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931} {"msg":"Crucible 56234a30-5d4b-4028-89b4-211985005a20 has session id: 0435b769-391b-450e-99fc-1dd879b8f976","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.94561455Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"Upstairs opts: Upstairs UUID: 56234a30-5d4b-4028-89b4-211985005a20, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.945659084Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"Crucible stats registered with UUID: 56234a30-5d4b-4028-89b4-211985005a20","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.94568498Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.9760954Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.976137673Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931} {"msg":"56234a30-5d4b-4028-89b4-211985005a20 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.976155551Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.976171968Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"client":"0","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.976205614Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"client":"0","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.976224071Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"client":"1","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.976241868Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"client":"1","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.976267974Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"client":"2","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.976350662Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"client":"2","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.976391096Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"":"io task","client":"2","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.976426031Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"":"io task","client":"0","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.976456297Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"":"io task","client":"1","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"ds_connection connected from Ok(127.0.0.1:34736)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.976551493Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"":"io task","client":"1","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"ds_connection connected from Ok(127.0.0.1:54829)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.976585558Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"":"io task","client":"2","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"ds_connection connected from Ok(127.0.0.1:49284)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.976610044Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"":"io task","client":"0","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.977364394Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"client":"0","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.977412317Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"client":"0","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.977447232Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"client":"1","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.977478538Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"client":"1","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.977513603Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"client":"2","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.977544268Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"client":"2","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.977971796Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"client":"1","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.978120404Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"client":"0","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.978265923Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"client":"2","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.978644027Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"client":"0","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.978671354Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"client":"0","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.978686271Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.978699359Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"client":"1","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.978713397Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"client":"1","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.978727105Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.978815062Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"client":"2","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.9788348Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"client":"2","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"[0]R flush_numbers[0..12]: [295, 272, 287, 294, 291, 291, 288, 293, 296, 296, 284, 295]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.978849337Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"":"downstairs","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"[0]R generation[0..12]: [184, 172, 181, 184, 182, 182, 182, 183, 185, 185, 179, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.978863446Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"":"downstairs","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.978881783Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"":"downstairs","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"[1]R flush_numbers[0..12]: [295, 272, 287, 294, 291, 291, 288, 293, 296, 296, 284, 295]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.978896521Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"":"downstairs","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"[1]R generation[0..12]: [184, 172, 181, 184, 182, 182, 182, 183, 185, 185, 179, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.978910359Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"":"downstairs","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.978924317Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"":"downstairs","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"[2]R flush_numbers[0..12]: [295, 272, 287, 294, 291, 291, 288, 293, 296, 296, 284, 295]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.978938235Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"":"downstairs","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"[2]R generation[0..12]: [184, 172, 181, 184, 182, 182, 182, 183, 185, 185, 179, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.978952093Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"":"downstairs","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.97896606Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"":"downstairs","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.978979369Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"":"downstairs","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.978992807Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"":"downstairs","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"Next flush: 297","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.979005935Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"":"downstairs","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.979019173Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"":"downstairs","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:37:32.979032631Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.979051868Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"client":"0","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.979065166Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"client":"1","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.979078484Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"client":"2","":"downstairs","session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.979094042Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.97910665Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"56234a30-5d4b-4028-89b4-211985005a20 is now active with session: 0435b769-391b-450e-99fc-1dd879b8f976","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.979119718Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.979131986Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931,"session_id":"0435b769-391b-450e-99fc-1dd879b8f976"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:32.979144394Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 496 len 36864 data: 13 12 14 16 17 17 13 14 13 02/30 Read block 522 len 40960 03/30 Read block 327 len 12288 04/30 Write block 285 len 40960 data: 13 13 13 13 14 9 8 11 12 12 05/30 Read block 592 len 20480 06/30 Write block 139 len 40960 data: 10 10 12 13 15 16 17 18 20 16 07/30 Read block 563 len 8192 08/30 Read block 403 len 16384 09/30 Write block 202 len 12288 data: 13 11 9 10/30 Read block 457 len 36864 11/30 Write block 587 len 20480 data: 15 15 18 17 14 12/30 Read block 509 len 40960 13/30 Write block 30 len 24576 data: 12 12 8 11 10 7 14/30 Write block 380 len 12288 data: 19 15 14 15/30 Write block 535 len 20480 data: 8 9 10 10 10 16/30 Read block 277 len 16384 17/30 Write block 331 len 36864 data: 10 9 10 12 10 11 11 13 12 18/30 Read block 546 len 8192 19/30 Write block 517 len 24576 data: 10 12 11 13 14 15 20/30 Read block 257 len 12288 21/30 Write block 86 len 4096 data: 17 22/30 Read block 322 len 4096 23/30 Write block 80 len 16384 data: 15 14 13 13 24/30 Write block 172 len 28672 data: 10 7 10 13 15 15 14 25/30 Read block 318 len 12288 26/30 Read block 27 len 28672 27/30 Read block 31 len 36864 28/30 Read block 521 len 28672 29/30 Write block 58 len 12288 data: 16 15 15 30/30 Write block 444 len 36864 data: 16 16 14 13 14 14 14 13 13 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Write 10 Done Done Done false 1004 Acked Read 5 Done Done Done false 1005 Acked Write 10 Done Done Done false 1006 Acked Read 2 Done Done Done false 1007 Acked Read 4 Done Done Done false 1008 Acked Write 3 Done Done Done false 1009 Acked Read 9 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Read 10 Done Done Done false 1012 Acked Write 6 Done Done Done false 1013 Acked Write 3 Done Done Done false 1014 Acked Write 5 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 3 Done Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 2 5 Done 29 28 28 85 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:32.999987461Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1931} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 295 295 295 F F F 1 020-039 172 172 172 272 272 272 T T T 2 040-059 181 181 181 287 287 287 T T T 3 060-079 184 184 184 294 294 294 T T T 4 080-099 182 182 182 291 291 291 T T T 5 100-119 182 182 182 291 291 291 F F F 6 120-139 182 182 182 288 288 288 T T T 7 140-159 183 183 183 293 293 293 T T T 8 160-179 185 185 185 296 296 296 T T T 9 180-199 185 185 185 296 296 296 F F F 10 200-219 179 179 179 284 284 284 T T T 11 220-239 184 184 184 295 295 295 F F F 12 240-259 172 172 172 271 271 271 F F F 13 260-279 184 184 184 294 294 294 F F F 14 280-299 180 180 180 286 286 286 T T T 15 300-319 180 180 180 286 286 286 F F F 16 320-339 184 184 184 295 295 295 T T T 17 340-359 184 184 184 295 295 295 F F F 18 360-379 185 185 185 296 296 296 F F F 19 380-399 185 185 185 296 296 296 T T T 20 400-419 185 185 185 296 296 296 F F F 21 420-439 184 184 184 294 294 294 F F F 22 440-459 183 183 183 293 293 293 T T T 23 460-479 182 182 182 292 292 292 F F F 24 480-499 184 184 184 294 294 294 T T T 25 500-519 184 184 184 294 294 294 T T T 26 520-539 184 184 184 295 295 295 T T T 27 540-559 185 185 185 296 296 296 F F F 28 560-579 182 182 182 292 292 292 F F F 29 580-599 171 171 171 269 269 269 T T T Max gen: 185, Max flush: 296 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 295 295 295 F F F 1 020-039 186 186 186 297 297 297 F F F 2 040-059 186 186 186 297 297 297 F F F 3 060-079 186 186 186 297 297 297 F F F 4 080-099 186 186 186 297 297 297 F F F 5 100-119 182 182 182 291 291 291 F F F 6 120-139 186 186 186 297 297 297 F F F 7 140-159 186 186 186 297 297 297 F F F 8 160-179 186 186 186 297 297 297 F F F 9 180-199 185 185 185 296 296 296 F F F 10 200-219 186 186 186 297 297 297 F F F 11 220-239 184 184 184 295 295 295 F F F 12 240-259 172 172 172 271 271 271 F F F 13 260-279 184 184 184 294 294 294 F F F 14 280-299 186 186 186 297 297 297 F F F 15 300-319 180 180 180 286 286 286 F F F 16 320-339 186 186 186 297 297 297 F F F 17 340-359 184 184 184 295 295 295 F F F 18 360-379 185 185 185 296 296 296 F F F 19 380-399 186 186 186 297 297 297 F F F 20 400-419 185 185 185 296 296 296 F F F 21 420-439 184 184 184 294 294 294 F F F 22 440-459 186 186 186 297 297 297 F F F 23 460-479 182 182 182 292 292 292 F F F 24 480-499 186 186 186 297 297 297 F F F 25 500-519 186 186 186 297 297 297 F F F 26 520-539 186 186 186 297 297 297 F F F 27 540-559 185 185 185 296 296 296 F F F 28 560-579 182 182 182 292 292 292 F F F 29 580-599 186 186 186 297 297 297 F F F Max gen: 186, Max flush: 297 Begin loop 94 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.335620389Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:37.377586892Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.377960337Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.409264986Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.409835283Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.409874207Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.409889635Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.409904453Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939} {"msg":"Crucible 77bacf4e-91c0-4893-8d3e-193d1b1d66c6 has session id: 36eebdf0-28fd-4fd2-869f-17f7f742d6eb","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.410352897Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"Upstairs opts: Upstairs UUID: 77bacf4e-91c0-4893-8d3e-193d1b1d66c6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.41040028Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"Crucible stats registered with UUID: 77bacf4e-91c0-4893-8d3e-193d1b1d66c6","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.410432586Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.440750199Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.440782644Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939} {"msg":"77bacf4e-91c0-4893-8d3e-193d1b1d66c6 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.44080833Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.440825398Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"client":"0","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.440853564Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"client":"0","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.440871971Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"client":"1","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.440890288Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"client":"1","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.440917414Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"client":"2","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.440951829Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"client":"2","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.440984804Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"":"io task","client":"2","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.44101893Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"":"io task","client":"0","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.441052315Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"":"io task","client":"1","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:60978)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.441099988Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"":"io task","client":"1","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:55909)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.441136892Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"":"io task","client":"0","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:59279)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.441257185Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"":"io task","client":"2","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.441842319Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"client":"1","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.441872515Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"client":"1","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.441900061Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"client":"2","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.441927847Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"client":"2","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.442002476Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"client":"0","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.442035701Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"client":"0","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.442478477Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"client":"1","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.442508572Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"client":"2","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.442808728Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"client":"0","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.44307161Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"client":"1","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.443100256Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"client":"1","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.443116393Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.443143919Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"client":"2","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.443159017Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"client":"2","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.443172535Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.443255513Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"client":"0","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.443278889Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"client":"0","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"[0]R flush_numbers[0..12]: [295, 297, 297, 297, 297, 291, 297, 297, 297, 296, 297, 295]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.443293778Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"":"downstairs","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"[0]R generation[0..12]: [184, 186, 186, 186, 186, 182, 186, 186, 186, 185, 186, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.443308275Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"":"downstairs","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.443322723Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"":"downstairs","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"[1]R flush_numbers[0..12]: [295, 297, 297, 297, 297, 291, 297, 297, 297, 296, 297, 295]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.443336691Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"":"downstairs","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"[1]R generation[0..12]: [184, 186, 186, 186, 186, 182, 186, 186, 186, 185, 186, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.443350459Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"":"downstairs","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.443364557Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"":"downstairs","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"[2]R flush_numbers[0..12]: [295, 297, 297, 297, 297, 291, 297, 297, 297, 296, 297, 295]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.443378495Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"":"downstairs","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"[2]R generation[0..12]: [184, 186, 186, 186, 186, 182, 186, 186, 186, 185, 186, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.443392203Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"":"downstairs","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.443406271Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"":"downstairs","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"Max found gen is 187","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.443419729Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"":"downstairs","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"Generation requested: 188 >= found:187","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.443433157Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"":"downstairs","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"Next flush: 298","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.443446285Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"":"downstairs","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-03T08:37:37.443459323Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"":"downstairs","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.44348289Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.443495178Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"client":"0","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.443508896Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"client":"1","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.443522384Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"client":"2","":"downstairs","session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.443535912Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.4435484Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"77bacf4e-91c0-4893-8d3e-193d1b1d66c6 is now active with session: 36eebdf0-28fd-4fd2-869f-17f7f742d6eb","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.443561408Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.443573557Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939,"session_id":"36eebdf0-28fd-4fd2-869f-17f7f742d6eb"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:37.443585975Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 456 len 36864 data: 16 21 20 18 17 17 15 11 11 02/30 Read block 591 len 36864 03/30 Read block 545 len 20480 04/30 Flush 05/30 Write block 522 len 8192 data: 16 17 06/30 Read block 299 len 12288 07/30 Write block 0 len 8192 data: 3 4 08/30 Write block 226 len 36864 data: 15 20 23 20 21 18 17 20 17 09/30 Read block 48 len 16384 10/30 Write block 19 len 20480 data: 17 14 15 12 11 11/30 Write block 135 len 36864 data: 13 10 9 8 11 11 13 14 16 12/30 Read block 307 len 12288 13/30 Read block 465 len 40960 14/30 Write block 83 len 32768 data: 14 13 21 18 14 15 16 17 15/30 Read block 452 len 20480 16/30 Read block 121 len 20480 17/30 Read block 159 len 16384 18/30 Read block 177 len 12288 19/30 Read block 17 len 12288 20/30 Write block 420 len 8192 data: 20 20 21/30 Write block 415 len 20480 data: 19 19 22 20 24 22/30 Write block 191 len 28672 data: 16 13 10 12 12 15 14 23/30 Write block 514 len 28672 data: 14 10 12 11 13 12 14 24/30 Write block 230 len 12288 data: 22 19 18 25/30 Write block 575 len 28672 data: 9 10 14 16 17 14 14 26/30 Write block 366 len 24576 data: 15 15 16 16 19 19 27/30 Write block 308 len 4096 data: 12 28/30 Write block 42 len 32768 data: 15 13 14 13 17 12 12 13 29/30 Read block 134 len 8192 30/30 Write block 459 len 36864 data: 19 18 18 16 12 12 10 12 13 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 2 Done Done Done false 1005 Acked Read 3 Done Done Done false 1006 Acked Write 2 Done Done Done false 1007 Acked Write 9 Done Done Done false 1008 Acked Read 4 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Write 9 Done Done Done false 1011 Acked Read 3 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 25 25 25 75 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:37.462631984Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1939} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 295 295 295 T T T 1 020-039 186 186 186 297 297 297 T T T 2 040-059 186 186 186 297 297 297 T T T 3 060-079 186 186 186 297 297 297 F F F 4 080-099 186 186 186 297 297 297 T T T 5 100-119 182 182 182 291 291 291 F F F 6 120-139 186 186 186 297 297 297 T T T 7 140-159 186 186 186 297 297 297 T T T 8 160-179 186 186 186 297 297 297 F F F 9 180-199 185 185 185 296 296 296 T T T 10 200-219 186 186 186 297 297 297 F F F 11 220-239 184 184 184 295 295 295 T T T 12 240-259 172 172 172 271 271 271 F F F 13 260-279 184 184 184 294 294 294 F F F 14 280-299 186 186 186 297 297 297 F F F 15 300-319 180 180 180 286 286 286 T T T 16 320-339 186 186 186 297 297 297 F F F 17 340-359 184 184 184 295 295 295 F F F 18 360-379 185 185 185 296 296 296 T T T 19 380-399 186 186 186 297 297 297 F F F 20 400-419 185 185 185 296 296 296 T T T 21 420-439 184 184 184 294 294 294 T T T 22 440-459 188 188 188 298 298 298 T T T 23 460-479 188 188 188 298 298 298 T T T 24 480-499 186 186 186 297 297 297 F F F 25 500-519 186 186 186 297 297 297 T T T 26 520-539 186 186 186 297 297 297 T T T 27 540-559 185 185 185 296 296 296 F F F 28 560-579 182 182 182 292 292 292 T T T 29 580-599 186 186 186 297 297 297 T T T Max gen: 188, Max flush: 298 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 299 299 299 F F F 1 020-039 189 189 189 299 299 299 F F F 2 040-059 189 189 189 299 299 299 F F F 3 060-079 186 186 186 297 297 297 F F F 4 080-099 189 189 189 299 299 299 F F F 5 100-119 182 182 182 291 291 291 F F F 6 120-139 189 189 189 299 299 299 F F F 7 140-159 189 189 189 299 299 299 F F F 8 160-179 186 186 186 297 297 297 F F F 9 180-199 189 189 189 299 299 299 F F F 10 200-219 186 186 186 297 297 297 F F F 11 220-239 189 189 189 299 299 299 F F F 12 240-259 172 172 172 271 271 271 F F F 13 260-279 184 184 184 294 294 294 F F F 14 280-299 186 186 186 297 297 297 F F F 15 300-319 189 189 189 299 299 299 F F F 16 320-339 186 186 186 297 297 297 F F F 17 340-359 184 184 184 295 295 295 F F F 18 360-379 189 189 189 299 299 299 F F F 19 380-399 186 186 186 297 297 297 F F F 20 400-419 189 189 189 299 299 299 F F F 21 420-439 189 189 189 299 299 299 F F F 22 440-459 189 189 189 299 299 299 F F F 23 460-479 189 189 189 299 299 299 F F F 24 480-499 186 186 186 297 297 297 F F F 25 500-519 189 189 189 299 299 299 F F F 26 520-539 189 189 189 299 299 299 F F F 27 540-559 185 185 185 296 296 296 F F F 28 560-579 189 189 189 299 299 299 F F F 29 580-599 189 189 189 299 299 299 F F F Max gen: 189, Max flush: 299 Begin loop 95 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.843357556Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.886718004Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.887315387Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.887351522Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.88736666Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.887381608Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947} {"msg":"Crucible 101f9fc8-f270-443b-8ddd-b7219927cdcd has session id: 0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.887808415Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"Upstairs opts: Upstairs UUID: 101f9fc8-f270-443b-8ddd-b7219927cdcd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.887856328Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"Crucible stats registered with UUID: 101f9fc8-f270-443b-8ddd-b7219927cdcd","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.887887444Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.918605088Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.918637254Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947} {"msg":"101f9fc8-f270-443b-8ddd-b7219927cdcd active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.918670909Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.918688456Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"client":"0","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.918714992Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"client":"0","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.91873321Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"client":"1","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.918751267Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"client":"1","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.918768645Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"client":"2","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.918785872Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"client":"2","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.918813968Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"":"io task","client":"2","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.918846583Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"":"io task","client":"0","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.918955837Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"":"io task","client":"1","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:33014)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.91900215Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"":"io task","client":"2","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:38396)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.919038685Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"":"io task","client":"0","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:63978)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.919178455Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"":"io task","client":"1","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.919821611Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"client":"2","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.919862685Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"client":"2","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.9198944Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"client":"0","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.919925386Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"client":"0","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.919946213Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"client":"1","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.919960391Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"client":"1","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.920403936Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"client":"2","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.920448429Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"client":"1","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.920593978Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"client":"0","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.921053941Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"client":"1","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.921083457Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"client":"1","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.921098504Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.921110993Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"client":"2","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.921124671Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"client":"2","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.921141988Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.921154586Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"client":"0","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.921168154Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"client":"0","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"[0]R flush_numbers[0..12]: [299, 299, 299, 297, 299, 291, 299, 299, 297, 299, 297, 299]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.921183662Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"":"downstairs","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"[0]R generation[0..12]: [189, 189, 189, 186, 189, 182, 189, 189, 186, 189, 186, 189]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.92119812Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"":"downstairs","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.921212678Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"":"downstairs","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"[1]R flush_numbers[0..12]: [299, 299, 299, 297, 299, 291, 299, 299, 297, 299, 297, 299]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.921226776Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"":"downstairs","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"[1]R generation[0..12]: [189, 189, 189, 186, 189, 182, 189, 189, 186, 189, 186, 189]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.921240654Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"":"downstairs","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.921254772Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"":"downstairs","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"[2]R flush_numbers[0..12]: [299, 299, 299, 297, 299, 291, 299, 299, 297, 299, 297, 299]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.92126889Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"":"downstairs","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"[2]R generation[0..12]: [189, 189, 189, 186, 189, 182, 189, 189, 186, 189, 186, 189]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.921282857Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"":"downstairs","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.921296975Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"":"downstairs","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-01-03T08:37:41.921310113Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"":"downstairs","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.92133379Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"":"downstairs","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"Next flush: 300","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.921349158Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"":"downstairs","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.921362676Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"":"downstairs","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.921376114Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.921388342Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"client":"0","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.9214015Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"client":"1","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.921415018Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"client":"2","":"downstairs","session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.921428506Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.921440954Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"101f9fc8-f270-443b-8ddd-b7219927cdcd is now active with session: 0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.921454012Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.921466411Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947,"session_id":"0ea68ee3-e4ce-41e4-99a8-bf845f8ce7e9"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:41.921479049Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 478 len 36864 data: 13 12 14 14 14 16 15 14 12 03/30 Write block 205 len 16384 data: 9 10 10 11 04/30 Write block 92 len 28672 data: 16 14 12 14 12 15 18 05/30 Read block 265 len 32768 06/30 Write block 189 len 20480 data: 16 17 17 14 11 07/30 Read block 541 len 40960 08/30 Read block 220 len 8192 09/30 Read block 340 len 36864 10/30 Read block 445 len 16384 11/30 Read block 567 len 32768 12/30 Read block 43 len 12288 13/30 Read block 217 len 20480 14/30 Write block 10 len 12288 data: 11 10 12 15/30 Write block 321 len 40960 data: 12 12 14 13 13 15 13 10 12 13 16/30 Read block 559 len 32768 17/30 Write block 241 len 36864 data: 19 19 22 24 25 22 22 22 20 18/30 Write block 442 len 36864 data: 16 16 17 17 15 14 15 15 15 19/30 Write block 56 len 12288 data: 12 15 17 20/30 Flush 21/30 Write block 75 len 8192 data: 18 20 22/30 Write block 320 len 28672 data: 13 13 13 15 14 14 16 23/30 Write block 2 len 8192 data: 6 6 24/30 Flush 25/30 Write block 202 len 8192 data: 14 12 26/30 Read block 342 len 36864 27/30 Read block 24 len 8192 28/30 Read block 492 len 12288 29/30 Read block 280 len 28672 30/30 Read block 461 len 36864 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 2 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 9 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 5 6 6 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:41.940848751Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1947} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 302 302 302 F F F 1 020-039 189 189 189 299 299 299 F F F 2 040-059 190 190 190 301 301 301 F F F 3 060-079 190 190 190 302 302 302 F F F 4 080-099 190 190 190 301 301 301 F F F 5 100-119 182 182 182 291 291 291 F F F 6 120-139 189 189 189 299 299 299 F F F 7 140-159 189 189 189 299 299 299 F F F 8 160-179 186 186 186 297 297 297 F F F 9 180-199 190 190 190 301 301 301 F F F 10 200-219 190 190 190 301 301 301 T T T 11 220-239 189 189 189 299 299 299 F F F 12 240-259 190 190 190 301 301 301 F F F 13 260-279 184 184 184 294 294 294 F F F 14 280-299 186 186 186 297 297 297 F F F 15 300-319 189 189 189 299 299 299 F F F 16 320-339 190 190 190 302 302 302 F F F 17 340-359 184 184 184 295 295 295 F F F 18 360-379 189 189 189 299 299 299 F F F 19 380-399 186 186 186 297 297 297 F F F 20 400-419 189 189 189 299 299 299 F F F 21 420-439 189 189 189 299 299 299 F F F 22 440-459 190 190 190 301 301 301 F F F 23 460-479 190 190 190 301 301 301 F F F 24 480-499 190 190 190 301 301 301 F F F 25 500-519 189 189 189 299 299 299 F F F 26 520-539 189 189 189 299 299 299 F F F 27 540-559 185 185 185 296 296 296 F F F 28 560-579 189 189 189 299 299 299 F F F 29 580-599 189 189 189 299 299 299 F F F Max gen: 190, Max flush: 302 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 302 302 302 F F F 1 020-039 189 189 189 299 299 299 F F F 2 040-059 190 190 190 301 301 301 F F F 3 060-079 190 190 190 302 302 302 F F F 4 080-099 190 190 190 301 301 301 F F F 5 100-119 182 182 182 291 291 291 F F F 6 120-139 189 189 189 299 299 299 F F F 7 140-159 189 189 189 299 299 299 F F F 8 160-179 186 186 186 297 297 297 F F F 9 180-199 190 190 190 301 301 301 F F F 10 200-219 191 191 191 303 303 303 F F F 11 220-239 189 189 189 299 299 299 F F F 12 240-259 190 190 190 301 301 301 F F F 13 260-279 184 184 184 294 294 294 F F F 14 280-299 186 186 186 297 297 297 F F F 15 300-319 189 189 189 299 299 299 F F F 16 320-339 190 190 190 302 302 302 F F F 17 340-359 184 184 184 295 295 295 F F F 18 360-379 189 189 189 299 299 299 F F F 19 380-399 186 186 186 297 297 297 F F F 20 400-419 189 189 189 299 299 299 F F F 21 420-439 189 189 189 299 299 299 F F F 22 440-459 190 190 190 301 301 301 F F F 23 460-479 190 190 190 301 301 301 F F F 24 480-499 190 190 190 301 301 301 F F F 25 500-519 189 189 189 299 299 299 F F F 26 520-539 189 189 189 299 299 299 F F F 27 540-559 185 185 185 296 296 296 F F F 28 560-579 189 189 189 299 299 299 F F F 29 580-599 189 189 189 299 299 299 F F F Max gen: 191, Max flush: 303 Begin loop 96 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.188401718Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:46.23077979Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.231143077Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.262724226Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.263407916Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.263443881Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.263458389Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.263472727Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955} {"msg":"Crucible afe2039b-9394-4001-a881-1ef1fa7a71a4 has session id: d56e5e1e-a95a-4f22-8d61-1cc7767232f7","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.26393296Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"Upstairs opts: Upstairs UUID: afe2039b-9394-4001-a881-1ef1fa7a71a4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.263971434Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"Crucible stats registered with UUID: afe2039b-9394-4001-a881-1ef1fa7a71a4","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.263987892Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.294166845Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.294210008Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955} {"msg":"afe2039b-9394-4001-a881-1ef1fa7a71a4 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.294282388Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.294318393Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"client":"0","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.294355237Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"client":"0","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.294388093Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"client":"1","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.294419818Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"client":"1","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.294452723Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"client":"2","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.294486198Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"client":"2","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.294518143Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"":"io task","client":"2","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.294548089Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"":"io task","client":"0","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.294576805Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"":"io task","client":"1","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:35715)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.294607881Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"":"io task","client":"1","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:53404)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.294688099Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"":"io task","client":"0","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:65168)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.294791334Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"":"io task","client":"2","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.295249347Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"client":"2","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.295282522Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"client":"2","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.29529936Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"client":"1","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.295314097Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"client":"1","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.295529386Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"client":"0","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.295556912Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"client":"0","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.295967092Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"client":"2","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.296001937Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"client":"1","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.296103772Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"client":"0","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.296624846Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"client":"1","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.296659691Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"client":"1","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.296688617Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.296705154Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"client":"2","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.296719362Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"client":"2","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.29673308Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.296745388Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"client":"0","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.296763036Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"client":"0","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"[0]R flush_numbers[0..12]: [302, 299, 301, 302, 301, 291, 299, 299, 297, 301, 303, 299]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.296777574Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"":"downstairs","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"[0]R generation[0..12]: [190, 189, 190, 190, 190, 182, 189, 189, 186, 190, 191, 189]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.296791682Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"":"downstairs","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.29680584Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"":"downstairs","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"[1]R flush_numbers[0..12]: [302, 299, 301, 302, 301, 291, 299, 299, 297, 301, 303, 299]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.296819827Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"":"downstairs","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"[1]R generation[0..12]: [190, 189, 190, 190, 190, 182, 189, 189, 186, 190, 191, 189]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.296833665Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"":"downstairs","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.296847774Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"":"downstairs","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"[2]R flush_numbers[0..12]: [302, 299, 301, 302, 301, 291, 299, 299, 297, 301, 303, 299]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.296861561Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"":"downstairs","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"[2]R generation[0..12]: [190, 189, 190, 190, 190, 182, 189, 189, 186, 190, 191, 189]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.296875279Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"":"downstairs","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.296952898Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"":"downstairs","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.296969046Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"":"downstairs","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.296982614Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"":"downstairs","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"Next flush: 304","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.296995642Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"":"downstairs","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.29700869Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"":"downstairs","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.297025937Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.297038515Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"client":"0","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.297052224Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"client":"1","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.297065492Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"client":"2","":"downstairs","session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.29707883Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.297091298Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 afe2039b-9394-4001-a881-1ef1fa7a71a4 is now active with session: d56e5e1e-a95a-4f22-8d61-1cc7767232f7","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.297107476Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.297120334Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955,"session_id":"d56e5e1e-a95a-4f22-8d61-1cc7767232f7"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:46.297133002Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 5 len 36864 data: 6 5 5 8 11 12 11 13 12 02/30 Read block 402 len 36864 03/30 Read block 239 len 40960 04/30 Flush 05/30 Write block 486 len 32768 data: 13 16 14 16 13 13 13 15 06/30 Read block 357 len 24576 07/30 Read block 172 len 40960 08/30 Read block 57 len 20480 09/30 Read block 544 len 16384 10/30 Write block 136 len 32768 data: 11 10 9 12 12 14 15 17 11/30 Write block 506 len 24576 data: 12 13 16 14 14 13 12/30 Read block 421 len 32768 13/30 Read block 292 len 32768 14/30 Read block 214 len 28672 15/30 Flush 16/30 Write block 171 len 36864 data: 12 11 8 11 14 16 16 15 15 17/30 Read block 462 len 28672 18/30 Write block 402 len 24576 data: 14 11 9 11 13 12 19/30 Read block 441 len 36864 20/30 Flush 21/30 Write block 263 len 40960 data: 19 16 14 17 16 16 13 12 14 12 22/30 Flush 23/30 Write block 424 len 20480 data: 11 12 9 10 10 24/30 Write block 139 len 4096 data: 13 25/30 Read block 185 len 32768 26/30 Write block 319 len 28672 data: 16 14 14 14 16 15 15 27/30 Read block 578 len 16384 28/30 Write block 476 len 32768 data: 17 15 14 13 15 15 15 17 29/30 Write block 197 len 4096 data: 15 30/30 Write block 97 len 20480 data: 16 19 22 18 17 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 5 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 8 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:46.317854636Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1955} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 304 304 304 F F F 1 020-039 189 189 189 299 299 299 F F F 2 040-059 190 190 190 301 301 301 F F F 3 060-079 190 190 190 302 302 302 F F F 4 080-099 190 190 190 301 301 301 T T T 5 100-119 182 182 182 291 291 291 T T T 6 120-139 192 192 192 305 305 305 T T T 7 140-159 192 192 192 305 305 305 F F F 8 160-179 192 192 192 306 306 306 F F F 9 180-199 190 190 190 301 301 301 T T T 10 200-219 191 191 191 303 303 303 F F F 11 220-239 189 189 189 299 299 299 F F F 12 240-259 190 190 190 301 301 301 F F F 13 260-279 192 192 192 307 307 307 F F F 14 280-299 186 186 186 297 297 297 F F F 15 300-319 189 189 189 299 299 299 T T T 16 320-339 190 190 190 302 302 302 T T T 17 340-359 184 184 184 295 295 295 F F F 18 360-379 189 189 189 299 299 299 F F F 19 380-399 186 186 186 297 297 297 F F F 20 400-419 192 192 192 306 306 306 F F F 21 420-439 189 189 189 299 299 299 T T T 22 440-459 190 190 190 301 301 301 F F F 23 460-479 190 190 190 301 301 301 T T T 24 480-499 192 192 192 305 305 305 T T T 25 500-519 192 192 192 305 305 305 F F F 26 520-539 189 189 189 299 299 299 F F F 27 540-559 185 185 185 296 296 296 F F F 28 560-579 189 189 189 299 299 299 F F F 29 580-599 189 189 189 299 299 299 F F F Max gen: 192, Max flush: 307 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 304 304 304 F F F 1 020-039 189 189 189 299 299 299 F F F 2 040-059 190 190 190 301 301 301 F F F 3 060-079 190 190 190 302 302 302 F F F 4 080-099 193 193 193 308 308 308 F F F 5 100-119 193 193 193 308 308 308 F F F 6 120-139 193 193 193 308 308 308 F F F 7 140-159 192 192 192 305 305 305 F F F 8 160-179 192 192 192 306 306 306 F F F 9 180-199 193 193 193 308 308 308 F F F 10 200-219 191 191 191 303 303 303 F F F 11 220-239 189 189 189 299 299 299 F F F 12 240-259 190 190 190 301 301 301 F F F 13 260-279 192 192 192 307 307 307 F F F 14 280-299 186 186 186 297 297 297 F F F 15 300-319 193 193 193 308 308 308 F F F 16 320-339 193 193 193 308 308 308 F F F 17 340-359 184 184 184 295 295 295 F F F 18 360-379 189 189 189 299 299 299 F F F 19 380-399 186 186 186 297 297 297 F F F 20 400-419 192 192 192 306 306 306 F F F 21 420-439 193 193 193 308 308 308 F F F 22 440-459 190 190 190 301 301 301 F F F 23 460-479 193 193 193 308 308 308 F F F 24 480-499 193 193 193 308 308 308 F F F 25 500-519 192 192 192 305 305 305 F F F 26 520-539 189 189 189 299 299 299 F F F 27 540-559 185 185 185 296 296 296 F F F 28 560-579 189 189 189 299 299 299 F F F 29 580-599 189 189 189 299 299 299 F F F Max gen: 193, Max flush: 308 Begin loop 97 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.639396231Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:50.681678957Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.682065501Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.71370878Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.714248421Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.714294525Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.714310542Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.71432602Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963} {"msg":"Crucible e9e0d22b-9bf0-4e10-8bd5-81fcbf45bd95 has session id: f828c9e8-97ec-414d-a1ec-a08c2ce9fd49","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.714760997Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"Upstairs opts: Upstairs UUID: e9e0d22b-9bf0-4e10-8bd5-81fcbf45bd95, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.71480749Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"Crucible stats registered with UUID: e9e0d22b-9bf0-4e10-8bd5-81fcbf45bd95","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.714834076Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.745428419Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.745466953Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963} {"msg":"e9e0d22b-9bf0-4e10-8bd5-81fcbf45bd95 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.745501598Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.745530384Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"client":"0","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.745573398Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"client":"0","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.745610222Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"client":"1","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.745645237Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"client":"1","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.745678512Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"client":"2","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.745713907Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"client":"2","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.745748822Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"":"io task","client":"2","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.745785177Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"":"io task","client":"0","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.745823601Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"":"io task","client":"1","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"ds_connection connected from Ok(127.0.0.1:61906)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.745919387Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"":"io task","client":"2","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"ds_connection connected from Ok(127.0.0.1:48753)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.745965101Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"":"io task","client":"0","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"ds_connection connected from Ok(127.0.0.1:32982)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.745992357Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"":"io task","client":"1","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.746491413Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"client":"1","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.746529638Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"client":"1","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.746562863Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"client":"2","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.746592829Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"client":"2","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.746673347Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"client":"0","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.746746276Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"client":"0","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.747152517Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"client":"1","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.747182473Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"client":"2","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.74719828Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"client":"0","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.747731552Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"client":"1","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.747757329Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"client":"1","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.747771986Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.747784875Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"client":"2","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.747798563Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"client":"2","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.747811991Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.747824159Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"client":"0","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.747837517Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"client":"0","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"[0]R flush_numbers[0..12]: [304, 299, 301, 302, 308, 308, 308, 305, 306, 308, 303, 299]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.747851455Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"":"downstairs","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"[0]R generation[0..12]: [192, 189, 190, 190, 193, 193, 193, 192, 192, 193, 191, 189]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.747868272Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"":"downstairs","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.74788267Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"":"downstairs","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"[1]R flush_numbers[0..12]: [304, 299, 301, 302, 308, 308, 308, 305, 306, 308, 303, 299]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.747896528Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"":"downstairs","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"[1]R generation[0..12]: [192, 189, 190, 190, 193, 193, 193, 192, 192, 193, 191, 189]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.747910436Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"":"downstairs","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.747924444Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"":"downstairs","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"[2]R flush_numbers[0..12]: [304, 299, 301, 302, 308, 308, 308, 305, 306, 308, 303, 299]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.747938262Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"":"downstairs","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"[2]R generation[0..12]: [192, 189, 190, 190, 193, 193, 193, 192, 192, 193, 191, 189]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.74795281Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"":"downstairs","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.747966908Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"":"downstairs","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.747979976Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"":"downstairs","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"Wait for a query_work_queue command to finish before sending IO Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.748001243Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"":"downstairs","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"Next flush: 309","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.748014401Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"":"downstairs","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.748027599Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"":"downstairs","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.748040927Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.748053156Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"client":"0","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.748068773Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"client":"1","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.748082521Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"client":"2","":"downstairs","session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.748095699Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.748108068Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"e9e0d22b-9bf0-4e10-8bd5-81fcbf45bd95 is now active with session: f828c9e8-97ec-414d-a1ec-a08c2ce9fd49","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.748120966Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.748133054Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963,"session_id":"f828c9e8-97ec-414d-a1ec-a08c2ce9fd49"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:50.748145332Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 18 len 40960 data: 14 18 15 16 13 12 13 12 10 11 02/30 Write block 391 len 12288 data: 14 15 13 03/30 Read block 228 len 20480 04/30 Write block 44 len 16384 data: 15 14 18 13 05/30 Read block 160 len 12288 06/30 Read block 378 len 8192 07/30 Read block 462 len 16384 08/30 Read block 332 len 8192 09/30 Read block 70 len 24576 10/30 Flush 11/30 Write block 493 len 32768 data: 16 13 14 14 13 15 17 18 12/30 Read block 96 len 24576 13/30 Read block 205 len 16384 14/30 Read block 259 len 4096 15/30 Flush 16/30 Write block 170 len 32768 data: 11 13 12 9 12 15 17 17 17/30 Read block 142 len 8192 18/30 Write block 246 len 8192 data: 23 23 19/30 Read block 40 len 8192 20/30 Read block 191 len 8192 21/30 Read block 538 len 36864 22/30 Read block 152 len 8192 23/30 Read block 429 len 36864 24/30 Read block 191 len 28672 25/30 Read block 111 len 24576 26/30 Write block 222 len 40960 data: 15 15 16 13 16 21 24 21 23 20 27/30 Read block 19 len 12288 28/30 Write block 258 len 24576 data: 20 18 18 18 19 20 29/30 Read block 1 len 28672 30/30 Write block 90 len 24576 data: 18 17 17 15 13 15 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 8 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:50.767652194Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1963} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 194 194 194 309 309 309 F F F 1 020-039 194 194 194 309 309 309 F F F 2 040-059 194 194 194 309 309 309 F F F 3 060-079 190 190 190 302 302 302 F F F 4 080-099 193 193 193 308 308 308 T T T 5 100-119 193 193 193 308 308 308 F F F 6 120-139 193 193 193 308 308 308 F F F 7 140-159 192 192 192 305 305 305 F F F 8 160-179 192 192 192 306 306 306 T T T 9 180-199 193 193 193 308 308 308 F F F 10 200-219 191 191 191 303 303 303 F F F 11 220-239 189 189 189 299 299 299 T T T 12 240-259 190 190 190 301 301 301 T T T 13 260-279 192 192 192 307 307 307 T T T 14 280-299 186 186 186 297 297 297 F F F 15 300-319 193 193 193 308 308 308 F F F 16 320-339 193 193 193 308 308 308 F F F 17 340-359 184 184 184 295 295 295 F F F 18 360-379 189 189 189 299 299 299 F F F 19 380-399 194 194 194 309 309 309 F F F 20 400-419 192 192 192 306 306 306 F F F 21 420-439 193 193 193 308 308 308 F F F 22 440-459 190 190 190 301 301 301 F F F 23 460-479 193 193 193 308 308 308 F F F 24 480-499 194 194 194 310 310 310 F F F 25 500-519 194 194 194 310 310 310 F F F 26 520-539 189 189 189 299 299 299 F F F 27 540-559 185 185 185 296 296 296 F F F 28 560-579 189 189 189 299 299 299 F F F 29 580-599 189 189 189 299 299 299 F F F Max gen: 194, Max flush: 310 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 194 194 194 309 309 309 F F F 1 020-039 194 194 194 309 309 309 F F F 2 040-059 194 194 194 309 309 309 F F F 3 060-079 190 190 190 302 302 302 F F F 4 080-099 195 195 195 311 311 311 F F F 5 100-119 193 193 193 308 308 308 F F F 6 120-139 193 193 193 308 308 308 F F F 7 140-159 192 192 192 305 305 305 F F F 8 160-179 195 195 195 311 311 311 F F F 9 180-199 193 193 193 308 308 308 F F F 10 200-219 191 191 191 303 303 303 F F F 11 220-239 195 195 195 311 311 311 F F F 12 240-259 195 195 195 311 311 311 F F F 13 260-279 195 195 195 311 311 311 F F F 14 280-299 186 186 186 297 297 297 F F F 15 300-319 193 193 193 308 308 308 F F F 16 320-339 193 193 193 308 308 308 F F F 17 340-359 184 184 184 295 295 295 F F F 18 360-379 189 189 189 299 299 299 F F F 19 380-399 194 194 194 309 309 309 F F F 20 400-419 192 192 192 306 306 306 F F F 21 420-439 193 193 193 308 308 308 F F F 22 440-459 190 190 190 301 301 301 F F F 23 460-479 193 193 193 308 308 308 F F F 24 480-499 194 194 194 310 310 310 F F F 25 500-519 194 194 194 310 310 310 F F F 26 520-539 189 189 189 299 299 299 F F F 27 540-559 185 185 185 296 296 296 F F F 28 560-579 189 189 189 299 299 299 F F F 29 580-599 189 189 189 299 299 299 F F F Max gen: 195, Max flush: 311 Begin loop 98 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.070242795Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:55.11328889Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.113656946Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.145787565Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.146455717Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.146497331Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.146514118Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.146531956Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971} {"msg":"Crucible e5526f34-be2b-431e-b652-455c9beb2572 has session id: 34822fd7-3b35-423b-b220-c251c8e7d305","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.146925118Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"Upstairs opts: Upstairs UUID: e5526f34-be2b-431e-b652-455c9beb2572, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.14698008Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"Crucible stats registered with UUID: e5526f34-be2b-431e-b652-455c9beb2572","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.147010796Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.177345826Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.177386731Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971} {"msg":"e5526f34-be2b-431e-b652-455c9beb2572 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.177454171Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.177479647Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"client":"0","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.177513112Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"client":"0","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.177534359Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"client":"1","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.177555606Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"client":"1","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.177575883Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"client":"2","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.17759625Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"client":"2","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.177616447Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"":"io task","client":"2","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.177637764Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"":"io task","client":"0","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.177658541Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"":"io task","client":"1","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"ds_connection connected from Ok(127.0.0.1:59724)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.178066791Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"":"io task","client":"2","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"ds_connection connected from Ok(127.0.0.1:44767)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.17814125Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"":"io task","client":"1","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"ds_connection connected from Ok(127.0.0.1:65070)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.178192993Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"":"io task","client":"0","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.178974409Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"client":"1","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.179003335Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"client":"1","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.179023672Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"client":"2","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.179039199Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"client":"2","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.17917483Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"client":"0","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.179215513Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"client":"0","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.179688535Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"client":"1","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.17971755Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"client":"0","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.179733478Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"client":"2","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.180239514Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"client":"2","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.18026828Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"client":"2","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.180283948Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.180369145Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"client":"1","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.180389132Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"client":"1","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.18040284Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.180415358Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"client":"0","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.180429376Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"client":"0","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"[0]R flush_numbers[0..12]: [309, 309, 309, 302, 311, 308, 308, 305, 311, 308, 303, 311]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.180443854Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"":"downstairs","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"[0]R generation[0..12]: [194, 194, 194, 190, 195, 193, 193, 192, 195, 193, 191, 195]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.180461922Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"":"downstairs","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.180477169Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"":"downstairs","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"[1]R flush_numbers[0..12]: [309, 309, 309, 302, 311, 308, 308, 305, 311, 308, 303, 311]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.180492217Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"":"downstairs","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"[1]R generation[0..12]: [194, 194, 194, 190, 195, 193, 193, 192, 195, 193, 191, 195]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.180506155Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"":"downstairs","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.180520033Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"":"downstairs","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"[2]R flush_numbers[0..12]: [309, 309, 309, 302, 311, 308, 308, 305, 311, 308, 303, 311]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.180533861Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"":"downstairs","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"[2]R generation[0..12]: [194, 194, 194, 190, 195, 193, 193, 192, 195, 193, 191, 195]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.180547619Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"":"downstairs","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.180561497Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.180582824Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"":"downstairs","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.180596122Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"":"downstairs","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"Next flush: 312","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.18060922Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"":"downstairs","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.180622458Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"":"downstairs","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.180635946Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.180647994Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"client":"0","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.180674011Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"client":"1","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.180689128Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"client":"2","":"downstairs","session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.180703366Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.180715794Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"e5526f34-be2b-431e-b652-455c9beb2572 is now active with session: 34822fd7-3b35-423b-b220-c251c8e7d305","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.180728752Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.180741371Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971,"session_id":"34822fd7-3b35-423b-b220-c251c8e7d305"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:55.180754069Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 179 len 28672 data: 16 12 12 15 18 21 20 02/30 Write block 503 len 20480 data: 15 14 13 13 14 03/30 Write block 431 len 28672 data: 6 10 8 9 13 16 11 04/30 Write block 440 len 12288 data: 14 16 17 05/30 Flush 06/30 Write block 582 len 28672 data: 14 15 15 12 14 16 16 07/30 Read block 413 len 16384 08/30 Write block 375 len 12288 data: 20 22 22 09/30 Read block 319 len 40960 10/30 Read block 234 len 36864 11/30 Read block 306 len 36864 12/30 Read block 160 len 20480 13/30 Read block 61 len 20480 14/30 Flush 15/30 Write block 176 len 28672 data: 18 18 16 17 13 13 16 16/30 Write block 66 len 8192 data: 17 18 17/30 Write block 336 len 32768 data: 12 12 14 13 11 13 13 11 18/30 Read block 132 len 4096 19/30 Read block 128 len 12288 20/30 Flush 21/30 Write block 554 len 12288 data: 15 16 13 22/30 Flush 23/30 Write block 38 len 32768 data: 10 15 13 11 16 14 16 15 24/30 Write block 126 len 4096 data: 12 25/30 Write block 355 len 16384 data: 9 9 10 8 26/30 Read block 80 len 4096 27/30 Read block 510 len 8192 28/30 Write block 28 len 32768 data: 12 10 13 13 9 12 11 8 29/30 Write block 161 len 20480 data: 21 22 22 23 21 30/30 Read block 356 len 4096 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 8 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 5 Sent Done Done false 1029 Acked Read 1 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 1 0 3 Done 6 7 8 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:55.19875992Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1971} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 194 194 194 309 309 309 F F F 1 020-039 194 194 194 309 309 309 T T T 2 040-059 194 194 194 309 309 309 T T T 3 060-079 196 196 196 314 314 314 F F F 4 080-099 195 195 195 311 311 311 F F F 5 100-119 193 193 193 308 308 308 F F F 6 120-139 193 193 193 308 308 308 T T T 7 140-159 192 192 192 305 305 305 F F F 8 160-179 196 196 196 314 314 314 T T T 9 180-199 196 196 196 314 314 314 F F F 10 200-219 191 191 191 303 303 303 F F F 11 220-239 195 195 195 311 311 311 F F F 12 240-259 195 195 195 311 311 311 F F F 13 260-279 195 195 195 311 311 311 F F F 14 280-299 186 186 186 297 297 297 F F F 15 300-319 193 193 193 308 308 308 F F F 16 320-339 196 196 196 314 314 314 F F F 17 340-359 196 196 196 314 314 314 T T T 18 360-379 196 196 196 313 313 313 F F F 19 380-399 194 194 194 309 309 309 F F F 20 400-419 192 192 192 306 306 306 F F F 21 420-439 196 196 196 312 312 312 F F F 22 440-459 196 196 196 312 312 312 F F F 23 460-479 193 193 193 308 308 308 F F F 24 480-499 194 194 194 310 310 310 F F F 25 500-519 196 196 196 312 312 312 F F F 26 520-539 189 189 189 299 299 299 F F F 27 540-559 196 196 196 315 315 315 F F F 28 560-579 189 189 189 299 299 299 F F F 29 580-599 196 196 196 313 313 313 F F F Max gen: 196, Max flush: 315 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 194 194 194 309 309 309 F F F 1 020-039 197 197 197 316 316 316 F F F 2 040-059 197 197 197 316 316 316 F F F 3 060-079 196 196 196 314 314 314 F F F 4 080-099 195 195 195 311 311 311 F F F 5 100-119 193 193 193 308 308 308 F F F 6 120-139 197 197 197 316 316 316 F F F 7 140-159 192 192 192 305 305 305 F F F 8 160-179 197 197 197 316 316 316 F F F 9 180-199 196 196 196 314 314 314 F F F 10 200-219 191 191 191 303 303 303 F F F 11 220-239 195 195 195 311 311 311 F F F 12 240-259 195 195 195 311 311 311 F F F 13 260-279 195 195 195 311 311 311 F F F 14 280-299 186 186 186 297 297 297 F F F 15 300-319 193 193 193 308 308 308 F F F 16 320-339 196 196 196 314 314 314 F F F 17 340-359 197 197 197 316 316 316 F F F 18 360-379 196 196 196 313 313 313 F F F 19 380-399 194 194 194 309 309 309 F F F 20 400-419 192 192 192 306 306 306 F F F 21 420-439 196 196 196 312 312 312 F F F 22 440-459 196 196 196 312 312 312 F F F 23 460-479 193 193 193 308 308 308 F F F 24 480-499 194 194 194 310 310 310 F F F 25 500-519 196 196 196 312 312 312 F F F 26 520-539 189 189 189 299 299 299 F F F 27 540-559 196 196 196 315 315 315 F F F 28 560-579 189 189 189 299 299 299 F F F 29 580-599 196 196 196 313 313 313 F F F Max gen: 197, Max flush: 316 Begin loop 99 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.502885536Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:59.545697636Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.54607987Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.57764957Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.578242773Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"94736ad28ed33108b1c88fccc3caea693da35514\",\n git_commit_timestamp: \"2025-01-03T08:14:58.000000000Z\",\n git_branch: \"renovate/tempfile-3.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.578285257Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.578302125Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.578321742Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979} {"msg":"Crucible a0b5812b-2505-4f62-9e1f-f351dd0d2c28 has session id: 72c66295-568b-4b15-8615-71a1a525d3db","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.578780175Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"Upstairs opts: Upstairs UUID: a0b5812b-2505-4f62-9e1f-f351dd0d2c28, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.57881714Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"Crucible stats registered with UUID: a0b5812b-2505-4f62-9e1f-f351dd0d2c28","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.578834787Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.608870691Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.608909236Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979} {"msg":"a0b5812b-2505-4f62-9e1f-f351dd0d2c28 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.608939521Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.608968117Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"client":"0","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.60901372Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"client":"0","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.609050175Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"client":"1","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.60908638Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"client":"1","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.609149301Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"client":"2","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.609187915Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"client":"2","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.60922097Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"":"io task","client":"2","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.609253945Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"":"io task","client":"0","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.60928913Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"":"io task","client":"1","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"ds_connection connected from Ok(127.0.0.1:57929)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.609444728Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"":"io task","client":"2","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"ds_connection connected from Ok(127.0.0.1:50867)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.609481592Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"":"io task","client":"0","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"ds_connection connected from Ok(127.0.0.1:53057)","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.609500789Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"":"io task","client":"1","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.609981109Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"client":"1","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.610008515Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"client":"1","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.610023293Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"client":"2","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.610037151Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"client":"2","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.610096062Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"client":"0","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.610124938Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"client":"0","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.610675338Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"client":"0","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.610704074Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"client":"2","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.610721851Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"client":"1","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.6113428Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"client":"1","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611377385Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"client":"1","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611392753Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611405671Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"client":"2","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611419269Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"client":"2","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611432587Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611445076Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"client":"0","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611458543Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"client":"0","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"[0]R flush_numbers[0..12]: [309, 316, 316, 314, 311, 308, 316, 305, 316, 314, 303, 311]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611472401Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"":"downstairs","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"[0]R generation[0..12]: [194, 197, 197, 196, 195, 193, 197, 192, 197, 196, 191, 195]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611486139Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"":"downstairs","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611503937Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"":"downstairs","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"[1]R flush_numbers[0..12]: [309, 316, 316, 314, 311, 308, 316, 305, 316, 314, 303, 311]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611518325Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"":"downstairs","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"[1]R generation[0..12]: [194, 197, 197, 196, 195, 193, 197, 192, 197, 196, 191, 195]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611532123Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"":"downstairs","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611546061Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"":"downstairs","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"[2]R flush_numbers[0..12]: [309, 316, 316, 314, 311, 308, 316, 305, 316, 314, 303, 311]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611559689Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"":"downstairs","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"[2]R generation[0..12]: [194, 197, 197, 196, 195, 193, 197, 192, 197, 196, 191, 195]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611573367Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"":"downstairs","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {Wait for a query_work_queue command to finish before sending IO "msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611638957Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"":"downstairs","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611668923Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"":"downstairs","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611683921Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"":"downstairs","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"Next flush: 317","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611697548Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"":"downstairs","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611711077Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"":"downstairs","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611724525Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611736843Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"client":"0","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611751231Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"client":"1","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611767848Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"client":"2","":"downstairs","session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611781616Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611794034Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"a0b5812b-2505-4f62-9e1f-f351dd0d2c28 is now active with session: 72c66295-568b-4b15-8615-71a1a525d3db","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611807072Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611819181Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979,"session_id":"72c66295-568b-4b15-8615-71a1a525d3db"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2025-01-03T08:37:59.611831759Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 225 len 16384 data: 14 17 22 25 03/30 Read block 79 len 28672 04/30 Write block 361 len 40960 data: 13 11 11 14 17 16 16 17 17 20 05/30 Flush 06/30 Write block 45 len 16384 data: 16 19 14 13 07/30 Write block 215 len 32768 data: 15 14 12 13 13 13 14 16 08/30 Write block 237 len 28672 data: 17 16 16 17 20 20 23 09/30 Read block 494 len 20480 10/30 Read block 170 len 32768 11/30 Flush 12/30 Flush 13/30 Flush 14/30 Write block 462 len 32768 data: 17 13 13 11 13 14 12 14 15/30 Read block 142 len 12288 16/30 Write block 16 len 36864 data: 15 15 15 19 16 17 14 13 14 17/30 Write block 573 len 24576 data: 13 7 10 11 15 17 18/30 Write block 384 len 20480 data: 10 8 9 10 7 19/30 Flush 20/30 Write block 118 len 8192 data: 20 21 21/30 Read block 529 len 36864 22/30 Write block 9 len 32768 data: 12 13 12 14 13 13 15 16 23/30 Read block 253 len 16384 24/30 Write block 194 len 8192 data: 13 13 25/30 Write block 355 len 40960 data: 10 10 11 9 10 10 14 12 12 15 26/30 Write block 432 len 20480 data: 11 9 10 14 17 27/30 Write block 331 len 40960 data: 11 10 11 13 11 13 13 15 14 12 28/30 Write block 56 len 28672 data: 13 16 18 16 16 12 14 29/30 Read block 403 len 40960 30/30 Write block 414 len 4096 data: 25 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 2 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 10 Done Done Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 10 10 9 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-03T08:37:59.631300696Z","hostname":"w-01JGNMH6YNCFP7Q0X4RS6MCSP5","pid":1979} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 198 198 198 322 322 322 T T T 1 020-039 198 198 198 322 322 322 F F F 2 040-059 198 198 198 319 319 319 T T T 3 060-079 196 196 196 314 314 314 T T T 4 080-099 195 195 195 311 311 311 F F F 5 100-119 193 193 193 308 308 308 T T T 6 120-139 197 197 197 316 316 316 F F F 7 140-159 192 192 192 305 305 305 F F F 8 160-179 197 197 197 316 316 316 F F F 9 180-199 196 196 196 314 314 314 T T T 10 200-219 198 198 198 319 319 319 F F F 11 220-239 198 198 198 319 319 319 F F F 12 240-259 198 198 198 319 319 319 F F F 13 260-279 195 195 195 311 311 311 F F F 14 280-299 186 186 186 297 297 297 F F F 15 300-319 193 193 193 308 308 308 F F F 16 320-339 196 196 196 314 314 314 T T T 17 340-359 197 197 197 316 316 316 T T T 18 360-379 198 198 198 318 318 318 T T T 19 380-399 198 198 198 322 322 322 F F F 20 400-419 192 192 192 306 306 306 T T T 21 420-439 196 196 196 312 312 312 T T T 22 440-459 196 196 196 312 312 312 F F F 23 460-479 198 198 198 322 322 322 F F F 24 480-499 194 194 194 310 310 310 F F F 25 500-519 196 196 196 312 312 312 F F F 26 520-539 189 189 189 299 299 299 F F F 27 540-559 196 196 196 315 315 315 F F F 28 560-579 198 198 198 322 322 322 F F F 29 580-599 196 196 196 313 313 313 F F F Max gen: 198, Max flush: 322 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 199 199 199 323 323 323 F F F 1 020-039 198 198 198 322 322 322 F F F 2 040-059 199 199 199 323 323 323 F F F 3 060-079 199 199 199 323 323 323 F F F 4 080-099 195 195 195 311 311 311 F F F 5 100-119 199 199 199 323 323 323 F F F 6 120-139 197 197 197 316 316 316 F F F 7 140-159 192 192 192 305 305 305 F F F 8 160-179 197 197 197 316 316 316 F F F 9 180-199 199 199 199 323 323 323 F F F 10 200-219 198 198 198 319 319 319 F F F 11 220-239 198 198 198 319 319 319 F F F 12 240-259 198 198 198 319 319 319 F F F 13 260-279 195 195 195 311 311 311 F F F 14 280-299 186 186 186 297 297 297 F F F 15 300-319 193 193 193 308 308 308 F F F 16 320-339 199 199 199 323 323 323 F F F 17 340-359 199 199 199 323 323 323 F F F 18 360-379 199 199 199 323 323 323 F F F 19 380-399 198 198 198 322 322 322 F F F 20 400-419 199 199 199 323 323 323 F F F 21 420-439 199 199 199 323 323 323 F F F 22 440-459 196 196 196 312 312 312 F F F 23 460-479 198 198 198 322 322 322 F F F 24 480-499 194 194 194 310 310 310 F F F 25 500-519 196 196 196 312 312 312 F F F 26 520-539 189 189 189 299 299 299 F F F 27 540-559 196 196 196 315 315 315 F F F 28 560-579 198 198 198 322 322 322 F F F 29 580-599 196 196 196 313 313 313 F F F Max gen: 199, Max flush: 323 7:20 Test duration Test completed real 7:20.003144303 user 55.935837906 sys 18.355621977 trap 0.079627652 tflt 0.030354237 dflt 0.075933117 kflt 0.006320528 lock 11:34:42.784607740 slp 35:16.448258011 lat 5.521108946 stop 0.060105799