Turn off color for downstairs dump Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.26119292 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.018560031 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.0189414 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1189 1190 1191 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.504507533Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.574492641Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.576160837Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.576244839Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.57626616Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.576306551Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193} {"msg":"Crucible e85ac87c-d1a8-4285-8037-4f7fdc1ce04f has session id: 7ff49717-3326-41fe-b96e-3474d354645e","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.577325663Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"Upstairs opts: Upstairs UUID: e85ac87c-d1a8-4285-8037-4f7fdc1ce04f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.577410444Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"Crucible stats registered with UUID: e85ac87c-d1a8-4285-8037-4f7fdc1ce04f","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.577432225Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.619473107Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.619511408Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193} {"msg":"e85ac87c-d1a8-4285-8037-4f7fdc1ce04f active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.619582539Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.61961464Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"client":"0","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.619689202Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"client":"0","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.619719372Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"client":"1","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.619744903Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"client":"1","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.619770983Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"client":"2","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.619797924Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"client":"2","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.619824985Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.619851715Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.619877646Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"ds_connection connected from Ok(127.0.0.1:33508)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.619903606Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"ds_connection connected from Ok(127.0.0.1:51022)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.619929057Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"ds_connection connected from Ok(127.0.0.1:60897)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.620266474Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.621886449Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"client":"2","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.62192819Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"client":"2","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.621960961Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"client":"0","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.621992942Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"client":"0","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.622024192Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"client":"1","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.622048863Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"client":"1","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.62331668Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"client":"1","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.623359101Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"client":"2","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.623393792Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"client":"0","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.623912503Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"client":"0","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.623954564Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"client":"0","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.623984325Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.624056276Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"client":"1","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.624096807Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"client":"1","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.624115268Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.624128838Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"client":"2","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.624142968Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"client":"2","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.624411764Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"":"downstairs","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.624431264Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"":"downstairs","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.624449285Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"":"downstairs","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.624519257Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"":"downstairs","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.624537917Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"":"downstairs","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.624552317Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"":"downstairs","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.624566198Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"":"downstairs","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.624580018Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"":"downstairs","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.624594248Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"":"downstairs","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.624607508Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"":"downstairs","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.624621109Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"":"downstairs","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.624634349Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"":"downstairs","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.624713571Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"":"downstairs","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.624731921Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.624753102Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"client":"0","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.624780302Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"client":"1","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.624806963Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"client":"2","":"downstairs","session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.624824923Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.624837873Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"e85ac87c-d1a8-4285-8037-4f7fdc1ce04f is now active with session: 7ff49717-3326-41fe-b96e-3474d354645e","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.624850914Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.624863134Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193,"session_id":"7ff49717-3326-41fe-b96e-3474d354645e"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:08.624875434Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 88 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:50:12.719989689Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1193} Begin loop 1 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.752414303Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.798265477Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.798970802Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.799024224Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.799054314Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.799079095Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195} {"msg":"Crucible 7fa9cbd6-f09d-41fa-a640-f0897c9961f3 has session id: d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.799537935Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"Upstairs opts: Upstairs UUID: 7fa9cbd6-f09d-41fa-a640-f0897c9961f3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.799642197Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"Crucible stats registered with UUID: 7fa9cbd6-f09d-41fa-a640-f0897c9961f3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.799669468Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.831012368Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.831040538Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195} {"msg":"7fa9cbd6-f09d-41fa-a640-f0897c9961f3 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.831100479Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.83112039Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"client":"0","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.8311453Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"client":"0","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.831161171Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"client":"1","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.831175871Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"client":"1","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.831199852Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"client":"2","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.831225982Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"client":"2","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.831253933Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"":"io task","client":"2","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.831286294Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"":"io task","client":"0","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.831315534Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"":"io task","client":"1","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"ds_connection connected from Ok(127.0.0.1:56157)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.831343715Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"":"io task","client":"2","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"ds_connection connected from Ok(127.0.0.1:43381)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.831371675Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"":"io task","client":"0","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"ds_connection connected from Ok(127.0.0.1:46166)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.831395896Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"":"io task","client":"1","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.832100431Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"client":"1","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.832127842Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"client":"1","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.832143842Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"client":"0","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.832157982Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"client":"0","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.832290775Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"client":"2","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.832335476Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"client":"2","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.832755585Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"client":"0","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.832798956Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"client":"1","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.832868158Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"client":"2","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.833377379Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"client":"1","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.83341133Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"client":"1","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.83342915Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.83344394Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"client":"0","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.833459401Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"client":"0","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.833474501Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.833488301Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"client":"2","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.833502962Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"client":"2","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.833518012Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"":"downstairs","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.833542503Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"":"downstairs","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.833560053Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"":"downstairs","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.833575083Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"":"downstairs","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.833589884Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"":"downstairs","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.833605304Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"":"downstairs","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.833620294Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"":"downstairs","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.833635225Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"":"downstairs","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.833652305Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"":"downstairs","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.833667275Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"":"downstairs","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.833681726Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"":"downstairs","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.833696076Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"":"downstairs","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:50:12.833710436Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"":"downstairs","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.833733727Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.833747387Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"client":"0","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.833761827Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"client":"1","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.833776237Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"client":"2","":"downstairs","session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.833790918Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.833804438Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"7fa9cbd6-f09d-41fa-a640-f0897c9961f3 is now active with session: d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.833818669Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.833832099Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195,"session_id":"d3b4fdcc-6719-47f4-baaf-c3e8a3c6e675"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:12.833845069Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 171 len 28672 data: 2 2 2 2 2 2 2 02/30 Read block 418 len 28672 03/30 Write block 197 len 12288 data: 2 2 2 04/30 Read block 474 len 20480 05/30 Write block 118 len 28672 data: 2 2 2 2 2 2 2 06/30 Read block 89 len 24576 07/30 Write block 377 len 24576 data: 2 2 2 2 2 2 08/30 Flush 09/30 Write block 299 len 16384 data: 2 2 2 2 10/30 Read block 48 len 20480 11/30 Write block 110 len 40960 data: 2 2 2 2 2 2 2 2 3 3 12/30 Write block 528 len 8192 data: 2 2 13/30 Read block 516 len 16384 14/30 Read block 562 len 24576 15/30 Write block 221 len 12288 data: 2 2 2 16/30 Read block 35 len 40960 17/30 Write block 186 len 4096 data: 2 18/30 Read block 343 len 28672 19/30 Flush 20/30 Flush 21/30 Write block 515 len 32768 data: 2 2 2 2 2 2 2 2 22/30 Write block 269 len 24576 data: 2 2 2 2 2 2 23/30 Read block 379 len 36864 24/30 Read block 98 len 28672 25/30 Flush 26/30 Write block 67 len 36864 data: 2 2 2 2 2 2 2 2 2 27/30 Write block 520 len 32768 data: 3 3 3 2 2 2 2 2 28/30 Write block 210 len 28672 data: 2 2 2 2 2 2 2 29/30 Read block 549 len 36864 30/30 Write block 234 len 32768 data: 2 2 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 9 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:50:12.854840054Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1195} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 1 1 1 1 1 1 T T T 4 080-099 1 1 1 1 1 1 F F F 5 100-119 2 2 2 3 3 3 F F F 6 120-139 2 2 2 2 2 2 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 2 2 2 F F F 9 180-199 2 2 2 3 3 3 F F F 10 200-219 1 1 1 1 1 1 T T T 11 220-239 2 2 2 3 3 3 T T T 12 240-259 1 1 1 1 1 1 T T T 13 260-279 2 2 2 5 5 5 F F F 14 280-299 2 2 2 3 3 3 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 2 2 2 2 2 2 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 2 2 2 5 5 5 F F F 26 520-539 2 2 2 5 5 5 T T T 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 5 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 3 3 3 6 6 6 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 2 2 2 3 3 3 F F F 6 120-139 2 2 2 2 2 2 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 2 2 2 F F F 9 180-199 2 2 2 3 3 3 F F F 10 200-219 3 3 3 6 6 6 F F F 11 220-239 3 3 3 6 6 6 F F F 12 240-259 3 3 3 6 6 6 F F F 13 260-279 2 2 2 5 5 5 F F F 14 280-299 2 2 2 3 3 3 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 2 2 2 2 2 2 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 2 2 2 5 5 5 F F F 26 520-539 3 3 3 6 6 6 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 6 Begin loop 2 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.121407879Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.165935395Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.166541118Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.166577069Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.166591529Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.16660596Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203} {"msg":"Crucible 37b40bf0-84cc-463b-b2ed-86ddb5858dc6 has session id: d1d69637-0a66-4af4-bb63-f7c1df886a0c","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.16708177Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"Upstairs opts: Upstairs UUID: 37b40bf0-84cc-463b-b2ed-86ddb5858dc6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.167129861Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"Crucible stats registered with UUID: 37b40bf0-84cc-463b-b2ed-86ddb5858dc6","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.167158861Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.198138374Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.198165574Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203} {"msg":"37b40bf0-84cc-463b-b2ed-86ddb5858dc6 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.198235936Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.198255326Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"client":"0","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.198280007Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"client":"0","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.198294997Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"client":"1","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.198314408Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"client":"1","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.198377369Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"client":"2","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.19845388Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"client":"2","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.198490791Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"":"io task","client":"2","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.198522182Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"":"io task","client":"0","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.198549813Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"":"io task","client":"1","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"ds_connection connected from Ok(127.0.0.1:46661)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.198575933Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"":"io task","client":"0","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"ds_connection connected from Ok(127.0.0.1:48136)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.198601894Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"":"io task","client":"2","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"ds_connection connected from Ok(127.0.0.1:64843)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.198663515Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"":"io task","client":"1","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.19934306Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"client":"2","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.1993715Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"client":"2","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.199386871Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"client":"0","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.199400911Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"client":"0","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.199477593Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"client":"1","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.199516003Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"client":"1","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.200019625Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"client":"2","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.200088596Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"client":"1","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.200119857Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"client":"0","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.200552766Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"client":"2","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.200575466Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"client":"2","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.200589587Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.200691179Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"client":"1","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.2007101Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"client":"1","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.20072486Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.20074131Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"client":"0","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.20075533Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"client":"0","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 6, 1, 3, 2, 1, 2, 3, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.200769041Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"":"downstairs","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 3, 1, 2, 2, 1, 2, 2, 3, 3]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.200782791Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"":"downstairs","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.200796871Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"":"downstairs","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 6, 1, 3, 2, 1, 2, 3, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.200810392Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"":"downstairs","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 3, 1, 2, 2, 1, 2, 2, 3, 3]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.200824162Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"":"downstairs","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.200838102Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"":"downstairs","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 6, 1, 3, 2, 1, 2, 3, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.200851523Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"":"downstairs","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 3, 1, 2, 2, 1, 2, 2, 3, 3]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.200864903Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"":"downstairs","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.200878853Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"":"downstairs","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.200892053Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"":"downstairs","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.200905314Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"":"downstairs","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"Next flush: 7","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:50:17.200918154Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"":"downstairs","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.200936974Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"":"downstairs","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.200952085Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.200964555Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"client":"0","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.200978005Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"client":"1","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.200991166Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"client":"2","":"downstairs","session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.201004546Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.201017036Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"37b40bf0-84cc-463b-b2ed-86ddb5858dc6 is now active with session: d1d69637-0a66-4af4-bb63-f7c1df886a0c","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.201029826Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.201042047Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203,"session_id":"d1d69637-0a66-4af4-bb63-f7c1df886a0c"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:17.201054407Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 545 len 8192 data: 2 2 02/30 Flush 03/30 Write block 506 len 40960 data: 2 2 2 2 2 2 2 2 2 3 04/30 Read block 356 len 24576 05/30 Write block 107 len 4096 data: 2 06/30 Write block 333 len 32768 data: 2 2 2 2 2 2 2 2 07/30 Read block 158 len 32768 08/30 Read block 300 len 20480 09/30 Write block 212 len 20480 data: 3 3 3 3 3 10/30 Write block 268 len 20480 data: 2 3 3 3 3 11/30 Write block 19 len 4096 data: 2 12/30 Flush 13/30 Write block 273 len 24576 data: 3 3 2 2 2 2 14/30 Flush 15/30 Write block 162 len 12288 data: 2 2 2 16/30 Read block 483 len 8192 17/30 Read block 557 len 4096 18/30 Read block 576 len 32768 19/30 Flush 20/30 Write block 144 len 32768 data: 2 2 2 2 2 2 2 2 21/30 Flush 22/30 Flush 23/30 Write block 413 len 8192 data: 2 2 24/30 Read block 76 len 12288 25/30 Read block 555 len 20480 26/30 Write block 594 len 24576 data: 2 2 2 2 2 2 27/30 Read block 483 len 32768 28/30 Read block 98 len 8192 29/30 Read block 409 len 28672 30/30 Read block 341 len 32768 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 2 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 8 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:50:17.22008113Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1203} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 3 3 3 6 6 6 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 4 4 4 8 8 8 F F F 6 120-139 2 2 2 2 2 2 F F F 7 140-159 4 4 4 11 11 11 F F F 8 160-179 4 4 4 10 10 10 F F F 9 180-199 2 2 2 3 3 3 F F F 10 200-219 4 4 4 8 8 8 F F F 11 220-239 3 3 3 6 6 6 F F F 12 240-259 3 3 3 6 6 6 F F F 13 260-279 4 4 4 9 9 9 F F F 14 280-299 2 2 2 3 3 3 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 4 4 4 8 8 8 F F F 17 340-359 4 4 4 8 8 8 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 2 2 2 2 2 2 F F F 20 400-419 1 1 1 1 1 1 T T T 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 4 4 4 8 8 8 F F F 26 520-539 3 3 3 6 6 6 F F F 27 540-559 4 4 4 7 7 7 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 T T T Max gen: 4, Max flush: 11 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 3 3 3 6 6 6 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 4 4 4 8 8 8 F F F 6 120-139 2 2 2 2 2 2 F F F 7 140-159 4 4 4 11 11 11 F F F 8 160-179 4 4 4 10 10 10 F F F 9 180-199 2 2 2 3 3 3 F F F 10 200-219 4 4 4 8 8 8 F F F 11 220-239 3 3 3 6 6 6 F F F 12 240-259 3 3 3 6 6 6 F F F 13 260-279 4 4 4 9 9 9 F F F 14 280-299 2 2 2 3 3 3 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 4 4 4 8 8 8 F F F 17 340-359 4 4 4 8 8 8 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 2 2 2 2 2 2 F F F 20 400-419 5 5 5 12 12 12 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 4 4 4 8 8 8 F F F 26 520-539 3 3 3 6 6 6 F F F 27 540-559 4 4 4 7 7 7 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 5 5 5 12 12 12 F F F Max gen: 5, Max flush: 12 Begin loop 3 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.467032369Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.511639246Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.51224797Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.51228394Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.512298161Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.512312681Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211} {"msg":"Crucible d3700642-459b-4d04-9460-7c3749262335 has session id: 27d42569-4adf-4357-add6-efded1df4b8d","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.512674689Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"Upstairs opts: Upstairs UUID: d3700642-459b-4d04-9460-7c3749262335, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.51272199Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"Crucible stats registered with UUID: d3700642-459b-4d04-9460-7c3749262335","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.512747041Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.543720673Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.543752293Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211} {"msg":"d3700642-459b-4d04-9460-7c3749262335 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.543778574Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.543796614Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"client":"0","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.543826985Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"client":"0","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.543850885Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"client":"1","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.543870016Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"client":"1","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.543896566Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"client":"2","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.543926387Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"client":"2","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.543956868Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"":"io task","client":"2","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.543989878Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"":"io task","client":"0","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.544024319Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"":"io task","client":"1","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:46653)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.54405773Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"":"io task","client":"2","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:36626)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.544228493Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"":"io task","client":"0","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:48079)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.544269965Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"":"io task","client":"1","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.544865077Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"client":"2","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.544893208Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"client":"2","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.544909438Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"client":"0","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.544924019Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"client":"0","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.544938199Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"client":"1","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.54497963Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"client":"1","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.545593013Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"client":"2","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.545622504Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"client":"0","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.545649524Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"client":"1","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.546173156Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"client":"1","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.546200556Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"client":"1","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.546215267Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.546288918Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"client":"0","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.546307559Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"client":"0","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.546320919Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.546334189Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"client":"2","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.54634773Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"client":"2","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"[0]R flush_numbers[0..12]: [8, 1, 1, 6, 1, 8, 2, 11, 10, 3, 8, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.5463617Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"":"downstairs","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"[0]R generation[0..12]: [4, 1, 1, 3, 1, 4, 2, 4, 4, 2, 4, 3]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.5463758Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"":"downstairs","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.546390251Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"":"downstairs","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"[1]R flush_numbers[0..12]: [8, 1, 1, 6, 1, 8, 2, 11, 10, 3, 8, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.546407571Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"":"downstairs","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"[1]R generation[0..12]: [4, 1, 1, 3, 1, 4, 2, 4, 4, 2, 4, 3]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.546421491Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"":"downstairs","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.546435401Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"":"downstairs","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"[2]R flush_numbers[0..12]: [8, 1, 1, 6, 1, 8, 2, 11, 10, 3, 8, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.546449052Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"":"downstairs","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"[2]R generation[0..12]: [4, 1, 1, 3, 1, 4, 2, 4, 4, 2, 4, 3]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.546462332Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"":"downstairs","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.546476312Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"":"downstairs","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.546489613Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"":"downstairs","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.546502983Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"":"downstairs","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"Next flush: 13","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.546516033Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"":"downstairs","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.546540814Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"":"downstairs","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.546554154Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.546566304Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"client":"0","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.546579555Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"client":"1","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.546593005Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"client":"2","":"downstairs","session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.546606295Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.546620255Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"d3700642-459b-4d04-9460-7c3749262335 is now active with session: 27d42569-4adf-4357-add6-efded1df4b8d","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.546633416Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.546645606Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211,"session_id":"27d42569-4adf-4357-add6-efded1df4b8d"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:21.546657966Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 197 len 8192 data: 3 3 02/30 Read block 236 len 4096 03/30 Read block 318 len 20480 04/30 Read block 111 len 4096 05/30 Read block 565 len 40960 06/30 Read block 223 len 20480 07/30 Read block 360 len 24576 08/30 Read block 88 len 4096 09/30 Write block 415 len 4096 data: 2 10/30 Read block 11 len 28672 11/30 Read block 351 len 36864 12/30 Write block 517 len 12288 data: 3 3 3 13/30 Flush 14/30 Write block 219 len 28672 data: 2 2 3 3 3 2 2 15/30 Read block 284 len 12288 16/30 Write block 354 len 24576 data: 2 2 2 2 2 2 17/30 Write block 335 len 16384 data: 3 3 3 3 18/30 Flush 19/30 Write block 569 len 32768 data: 2 2 2 2 2 2 2 2 20/30 Read block 411 len 28672 21/30 Read block 367 len 12288 22/30 Read block 530 len 28672 23/30 Read block 512 len 28672 24/30 Write block 405 len 20480 data: 2 2 2 2 2 25/30 Read block 250 len 8192 26/30 Write block 152 len 4096 data: 2 27/30 Write block 44 len 36864 data: 2 2 2 2 2 2 2 2 2 28/30 Read block 176 len 32768 29/30 Write block 463 len 12288 data: 2 2 2 30/30 Read block 533 len 36864 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 8 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Read 9 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 12 12 11 35 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:50:21.565473324Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1211} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 T T T 3 060-079 3 3 3 6 6 6 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 4 4 4 8 8 8 F F F 6 120-139 2 2 2 2 2 2 F F F 7 140-159 4 4 4 11 11 11 T T T 8 160-179 4 4 4 10 10 10 F F F 9 180-199 6 6 6 13 13 13 F F F 10 200-219 6 6 6 14 14 14 F F F 11 220-239 6 6 6 14 14 14 F F F 12 240-259 3 3 3 6 6 6 F F F 13 260-279 4 4 4 9 9 9 F F F 14 280-299 2 2 2 3 3 3 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 6 6 6 14 14 14 F F F 17 340-359 6 6 6 14 14 14 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 2 2 2 2 2 2 F F F 20 400-419 6 6 6 13 13 13 T T T 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 T T T 24 480-499 1 1 1 1 1 1 F F F 25 500-519 6 6 6 13 13 13 F F F 26 520-539 3 3 3 6 6 6 F F F 27 540-559 4 4 4 7 7 7 F F F 28 560-579 1 1 1 1 1 1 T T T 29 580-599 5 5 5 12 12 12 F F F Max gen: 6, Max flush: 14 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 7 7 7 15 15 15 F F F 3 060-079 3 3 3 6 6 6 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 4 4 4 8 8 8 F F F 6 120-139 2 2 2 2 2 2 F F F 7 140-159 7 7 7 15 15 15 F F F 8 160-179 4 4 4 10 10 10 F F F 9 180-199 6 6 6 13 13 13 F F F 10 200-219 6 6 6 14 14 14 F F F 11 220-239 6 6 6 14 14 14 F F F 12 240-259 3 3 3 6 6 6 F F F 13 260-279 4 4 4 9 9 9 F F F 14 280-299 2 2 2 3 3 3 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 6 6 6 14 14 14 F F F 17 340-359 6 6 6 14 14 14 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 2 2 2 2 2 2 F F F 20 400-419 7 7 7 15 15 15 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 7 7 7 15 15 15 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 6 6 6 13 13 13 F F F 26 520-539 3 3 3 6 6 6 F F F 27 540-559 4 4 4 7 7 7 F F F 28 560-579 7 7 7 15 15 15 F F F 29 580-599 5 5 5 12 12 12 F F F Max gen: 7, Max flush: 15 Begin loop 4 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.828020752Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:50:25.870848971Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.871204358Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.903108571Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.903695143Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.903730924Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.903745924Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.903761965Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219} {"msg":"Crucible a6da6b9c-5ef4-4bed-9e3e-73c6e4dc4181 has session id: 222da8bf-4b3d-48bc-a9a8-847cb01bec21","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.904139393Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"Upstairs opts: Upstairs UUID: a6da6b9c-5ef4-4bed-9e3e-73c6e4dc4181, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.904184174Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"Crucible stats registered with UUID: a6da6b9c-5ef4-4bed-9e3e-73c6e4dc4181","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.904208374Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.934989402Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.935022163Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219} {"msg":"a6da6b9c-5ef4-4bed-9e3e-73c6e4dc4181 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.935089995Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.935113975Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"client":"0","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.935143456Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"client":"0","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.935163006Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"client":"1","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.935181166Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"client":"1","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.935203767Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"client":"2","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.935222367Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"client":"2","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.935250158Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"":"io task","client":"2","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.935282929Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"":"io task","client":"0","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.935316639Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"":"io task","client":"1","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"ds_connection connected from Ok(127.0.0.1:33900)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.93535399Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"":"io task","client":"2","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"ds_connection connected from Ok(127.0.0.1:49170)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.935388981Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"":"io task","client":"1","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"ds_connection connected from Ok(127.0.0.1:36881)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.935619836Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"":"io task","client":"0","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.936153507Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"client":"1","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.936192008Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"client":"1","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.936223889Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"client":"2","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.9362528Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"client":"2","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.93627906Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"client":"0","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.936294021Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"client":"0","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.936872703Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"client":"2","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.936919314Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"client":"1","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.936954375Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"client":"0","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.937483986Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"client":"2","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.937509317Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"client":"2","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.937523687Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.937550658Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"client":"1","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.937565218Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"client":"1","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.937578588Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.937590869Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"client":"0","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.937604449Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"client":"0","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"[0]R flush_numbers[0..12]: [8, 1, 15, 6, 1, 8, 2, 15, 10, 13, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.937618829Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"":"downstairs","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"[0]R generation[0..12]: [4, 1, 7, 3, 1, 4, 2, 7, 4, 6, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.937632579Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"":"downstairs","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.93764671Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"":"downstairs","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"[1]R flush_numbers[0..12]: [8, 1, 15, 6, 1, 8, 2, 15, 10, 13, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.93766053Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"":"downstairs","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"[1]R generation[0..12]: [4, 1, 7, 3, 1, 4, 2, 7, 4, 6, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.93767416Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"":"downstairs","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.937691381Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"":"downstairs","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"[2]R flush_numbers[0..12]: [8, 1, 15, 6, 1, 8, 2, 15, 10, 13, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.937705201Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"":"downstairs","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"[2]R generation[0..12]: [4, 1, 7, 3, 1, 4, 2, 7, 4, 6, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.937718781Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"":"downstairs","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.937732762Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"":"downstairs","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"Max found gen is 8"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.937810974Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"":"downstairs","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.937843624Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"":"downstairs","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"Next flush: 16","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.937858964Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"":"downstairs","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.937872475Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"":"downstairs","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.937890815Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.937914236Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"client":"0","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.937942056Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"client":"1","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.937957607Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"client":"2","":"downstairs","session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.937981147Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.938005978Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"a6da6b9c-5ef4-4bed-9e3e-73c6e4dc4181 is now active with session: 222da8bf-4b3d-48bc-a9a8-847cb01bec21","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.938032678Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.938058519Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219,"session_id":"222da8bf-4b3d-48bc-a9a8-847cb01bec21"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:25.938073129Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 544 len 36864 data: 2 3 3 2 2 2 2 2 2 02/30 Write block 128 len 36864 data: 2 2 2 2 2 2 2 2 2 03/30 Read block 219 len 28672 04/30 Read block 229 len 4096 05/30 Write block 467 len 16384 data: 2 2 2 2 06/30 Read block 364 len 32768 07/30 Read block 203 len 40960 08/30 Write block 15 len 24576 data: 2 2 2 2 3 2 09/30 Write block 200 len 16384 data: 2 2 2 2 10/30 Write block 186 len 40960 data: 3 2 2 2 2 2 2 2 2 2 11/30 Flush 12/30 Write block 115 len 36864 data: 3 3 3 4 4 3 3 3 3 13/30 Read block 11 len 8192 14/30 Read block 425 len 36864 15/30 Read block 497 len 36864 16/30 Read block 165 len 40960 17/30 Write block 207 len 40960 data: 2 2 2 3 3 4 4 4 4 4 18/30 Read block 60 len 4096 19/30 Read block 389 len 32768 20/30 Read block 1 len 28672 21/30 Write block 69 len 28672 data: 3 3 3 3 3 3 3 22/30 Write block 183 len 28672 data: 2 2 2 4 3 3 3 23/30 Write block 533 len 8192 data: 2 2 24/30 Write block 69 len 36864 data: 4 4 4 4 4 4 4 2 2 25/30 Read block 313 len 36864 26/30 Write block 490 len 24576 data: 2 2 2 2 2 2 27/30 Read block 408 len 12288 28/30 Write block 365 len 32768 data: 2 2 2 2 2 2 2 2 29/30 Write block 56 len 20480 data: 2 2 2 2 2 30/30 Read block 358 len 32768 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 9 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 19 18 18 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:50:25.958769228Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1219} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 16 16 16 F F F 1 020-039 8 8 8 16 16 16 F F F 2 040-059 7 7 7 15 15 15 T T T 3 060-079 3 3 3 6 6 6 T T T 4 080-099 1 1 1 1 1 1 F F F 5 100-119 4 4 4 8 8 8 T T T 6 120-139 8 8 8 16 16 16 T T T 7 140-159 7 7 7 15 15 15 F F F 8 160-179 4 4 4 10 10 10 F F F 9 180-199 8 8 8 16 16 16 T T T 10 200-219 8 8 8 16 16 16 T T T 11 220-239 6 6 6 14 14 14 F F F 12 240-259 3 3 3 6 6 6 F F F 13 260-279 4 4 4 9 9 9 F F F 14 280-299 2 2 2 3 3 3 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 6 6 6 14 14 14 F F F 17 340-359 6 6 6 14 14 14 F F F 18 360-379 2 2 2 2 2 2 T T T 19 380-399 2 2 2 2 2 2 F F F 20 400-419 7 7 7 15 15 15 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 8 8 8 16 16 16 F F F 24 480-499 1 1 1 1 1 1 T T T 25 500-519 6 6 6 13 13 13 F F F 26 520-539 3 3 3 6 6 6 T T T 27 540-559 8 8 8 16 16 16 F F F 28 560-579 7 7 7 15 15 15 F F F 29 580-599 5 5 5 12 12 12 F F F Max gen: 8, Max flush: 16 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 16 16 16 F F F 1 020-039 8 8 8 16 16 16 F F F 2 040-059 9 9 9 17 17 17 F F F 3 060-079 9 9 9 17 17 17 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 9 9 9 17 17 17 F F F 6 120-139 9 9 9 17 17 17 F F F 7 140-159 7 7 7 15 15 15 F F F 8 160-179 4 4 4 10 10 10 F F F 9 180-199 9 9 9 17 17 17 F F F 10 200-219 9 9 9 17 17 17 F F F 11 220-239 6 6 6 14 14 14 F F F 12 240-259 3 3 3 6 6 6 F F F 13 260-279 4 4 4 9 9 9 F F F 14 280-299 2 2 2 3 3 3 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 6 6 6 14 14 14 F F F 17 340-359 6 6 6 14 14 14 F F F 18 360-379 9 9 9 17 17 17 F F F 19 380-399 2 2 2 2 2 2 F F F 20 400-419 7 7 7 15 15 15 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 8 8 8 16 16 16 F F F 24 480-499 9 9 9 17 17 17 F F F 25 500-519 6 6 6 13 13 13 F F F 26 520-539 9 9 9 17 17 17 F F F 27 540-559 8 8 8 16 16 16 F F F 28 560-579 7 7 7 15 15 15 F F F 29 580-599 5 5 5 12 12 12 F F F Max gen: 9, Max flush: 17 Begin loop 5 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.274756635Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.31879804Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.319374673Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.319414154Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.319434064Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.319450974Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227} {"msg":"Crucible 49af5408-34d3-4d87-afe6-c08ad8ad031d has session id: 32be7e45-d512-43f7-9a82-b6c55c65e121","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.319963536Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"Upstairs opts: Upstairs UUID: 49af5408-34d3-4d87-afe6-c08ad8ad031d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.320000677Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"Crucible stats registered with UUID: 49af5408-34d3-4d87-afe6-c08ad8ad031d","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.320017647Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.35106597Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.351093391Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227} {"msg":"49af5408-34d3-4d87-afe6-c08ad8ad031d active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.351158732Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.351177003Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"client":"0","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.351200153Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"client":"0","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.351226614Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"client":"1","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.351254644Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"client":"1","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.351283725Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"client":"2","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.351353486Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"client":"2","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.351401258Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"":"io task","client":"2","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.351435438Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"":"io task","client":"0","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.35152879Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"":"io task","client":"1","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"ds_connection connected from Ok(127.0.0.1:61138)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.351575742Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"":"io task","client":"2","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"ds_connection connected from Ok(127.0.0.1:43326)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.351612532Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"":"io task","client":"0","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"ds_connection connected from Ok(127.0.0.1:58425)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.351645233Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"":"io task","client":"1","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.352203095Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"client":"2","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.352245286Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"client":"2","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.352275106Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"client":"0","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.352304267Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"client":"0","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.352334618Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"client":"1","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.352363439Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"client":"1","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.352818988Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"client":"0","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.352982482Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"client":"1","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.353061354Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"client":"2","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.353314549Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"client":"0","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.35334453Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"client":"0","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.35336164Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.353502113Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"client":"1","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.353522084Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"client":"1","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.353547294Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.353605435Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"client":"2","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.353624346Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"client":"2","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"[0]R flush_numbers[0..12]: [16, 16, 17, 17, 1, 17, 17, 15, 10, 17, 17, 14]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.353638646Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"":"downstairs","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"[0]R generation[0..12]: [8, 8, 9, 9, 1, 9, 9, 7, 4, 9, 9, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.353653256Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"":"downstairs","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.353667557Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"":"downstairs","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"[1]R flush_numbers[0..12]: [16, 16, 17, 17, 1, 17, 17, 15, 10, 17, 17, 14]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.353681237Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"":"downstairs","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"[1]R generation[0..12]: [8, 8, 9, 9, 1, 9, 9, 7, 4, 9, 9, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.353694857Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"":"downstairs","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.353708908Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"":"downstairs","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"[2]R flush_numbers[0..12]: [16, 16, 17, 17, 1, 17, 17, 15, 10, 17, 17, 14]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.353726698Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"":"downstairs","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"[2]R generation[0..12]: [8, 8, 9, 9, 1, 9, 9, 7, 4, 9, 9, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.353741048Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"":"downstairs","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.353755309Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"":"downstairs","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.353769459Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"":"downstairs","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.353783019Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"":"downstairs","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"Next flush: 18","v":0Wait for a query_work_queue command to finish before sending IO ,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.35380361Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"":"downstairs","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.35381787Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"":"downstairs","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.35383146Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.353843711Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"client":"0","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.353857191Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"client":"1","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.353871321Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"client":"2","":"downstairs","session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.353884901Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.353898062Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"49af5408-34d3-4d87-afe6-c08ad8ad031d is now active with session: 32be7e45-d512-43f7-9a82-b6c55c65e121","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.353911042Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.353923112Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227,"session_id":"32be7e45-d512-43f7-9a82-b6c55c65e121"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:30.353935503Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 201 len 16384 data: 3 3 3 2 03/30 Flush 04/30 Write block 500 len 4096 data: 2 05/30 Read block 374 len 4096 06/30 Write block 435 len 28672 data: 2 2 2 2 2 2 2 07/30 Write block 398 len 8192 data: 2 2 08/30 Read block 39 len 36864 09/30 Read block 357 len 24576 10/30 Flush 11/30 Write block 85 len 36864 data: 2 2 2 2 2 2 2 2 2 12/30 Write block 343 len 40960 data: 2 2 2 2 2 2 2 2 2 2 13/30 Write block 336 len 20480 data: 4 4 4 3 3 14/30 Write block 503 len 20480 data: 2 2 2 3 3 15/30 Write block 371 len 40960 data: 3 3 2 2 2 2 3 3 3 3 16/30 Write block 145 len 4096 data: 3 17/30 Read block 124 len 16384 18/30 Flush 19/30 Write block 102 len 16384 data: 2 2 2 2 20/30 Flush 21/30 Write block 284 len 12288 data: 2 2 2 22/30 Read block 43 len 16384 23/30 Read block 93 len 12288 24/30 Write block 62 len 40960 data: 2 2 2 2 2 3 3 5 5 5 25/30 Write block 94 len 4096 data: 2 26/30 Write block 534 len 28672 data: 3 2 2 2 2 2 2 27/30 Write block 229 len 4096 data: 2 28/30 Write block 142 len 8192 data: 2 2 29/30 Write block 168 len 8192 data: 2 2 30/30 Read block 390 len 8192 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 3 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 2 Done Sent Done false 1029 Acked Read 2 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 0 2 Done 10 8 10 28 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:50:30.370377309Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1227} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 16 16 16 F F F 1 020-039 8 8 8 16 16 16 F F F 2 040-059 9 9 9 17 17 17 F F F 3 060-079 9 9 9 17 17 17 T T T 4 080-099 10 10 10 21 21 21 T T T 5 100-119 10 10 10 22 22 22 F F F 6 120-139 9 9 9 17 17 17 F F F 7 140-159 10 10 10 21 21 21 T T T 8 160-179 4 4 4 10 10 10 T T T 9 180-199 9 9 9 17 17 17 F F F 10 200-219 10 10 10 19 19 19 F F F 11 220-239 6 6 6 14 14 14 T T T 12 240-259 3 3 3 6 6 6 F F F 13 260-279 4 4 4 9 9 9 F F F 14 280-299 2 2 2 3 3 3 T T T 15 300-319 2 2 2 3 3 3 F F F 16 320-339 10 10 10 21 21 21 F F F 17 340-359 10 10 10 21 21 21 F F F 18 360-379 10 10 10 21 21 21 F F F 19 380-399 10 10 10 21 21 21 F F F 20 400-419 7 7 7 15 15 15 F F F 21 420-439 10 10 10 20 20 20 F F F 22 440-459 10 10 10 20 20 20 F F F 23 460-479 8 8 8 16 16 16 F F F 24 480-499 9 9 9 17 17 17 F F F 25 500-519 10 10 10 21 21 21 F F F 26 520-539 9 9 9 17 17 17 T T T 27 540-559 8 8 8 16 16 16 T T T 28 560-579 7 7 7 15 15 15 F F F 29 580-599 5 5 5 12 12 12 F F F Max gen: 10, Max flush: 22 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 16 16 16 F F F 1 020-039 8 8 8 16 16 16 F F F 2 040-059 9 9 9 17 17 17 F F F 3 060-079 11 11 11 23 23 23 F F F 4 080-099 11 11 11 23 23 23 F F F 5 100-119 10 10 10 22 22 22 F F F 6 120-139 9 9 9 17 17 17 F F F 7 140-159 11 11 11 23 23 23 F F F 8 160-179 11 11 11 23 23 23 F F F 9 180-199 9 9 9 17 17 17 F F F 10 200-219 10 10 10 19 19 19 F F F 11 220-239 11 11 11 23 23 23 F F F 12 240-259 3 3 3 6 6 6 F F F 13 260-279 4 4 4 9 9 9 F F F 14 280-299 11 11 11 23 23 23 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 10 10 10 21 21 21 F F F 17 340-359 10 10 10 21 21 21 F F F 18 360-379 10 10 10 21 21 21 F F F 19 380-399 10 10 10 21 21 21 F F F 20 400-419 7 7 7 15 15 15 F F F 21 420-439 10 10 10 20 20 20 F F F 22 440-459 10 10 10 20 20 20 F F F 23 460-479 8 8 8 16 16 16 F F F 24 480-499 9 9 9 17 17 17 F F F 25 500-519 10 10 10 21 21 21 F F F 26 520-539 11 11 11 23 23 23 F F F 27 540-559 11 11 11 23 23 23 F F F 28 560-579 7 7 7 15 15 15 F F F 29 580-599 5 5 5 12 12 12 F F F Max gen: 11, Max flush: 23 Begin loop 6 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.651188023Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:50:34.694981303Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.695366451Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.726903025Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.727453697Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.727488818Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.727506919Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.727522179Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235} {"msg":"Crucible dc27fe62-289b-4131-9310-b3d3d1efdd7c has session id: 11b201fe-3eac-4cd0-844d-29ba82ff46a3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.727941348Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"Upstairs opts: Upstairs UUID: dc27fe62-289b-4131-9310-b3d3d1efdd7c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.727993269Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"Crucible stats registered with UUID: dc27fe62-289b-4131-9310-b3d3d1efdd7c","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.72801475Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.758828048Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.758873279Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235} {"msg":"dc27fe62-289b-4131-9310-b3d3d1efdd7c active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.75890378Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.758935651Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"client":"0","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.758986481Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"client":"0","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.759024352Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"client":"1","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.759059393Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"client":"1","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.759089574Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"client":"2","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.759118544Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"client":"2","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.759144205Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"":"io task","client":"0","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.759168105Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.759183516Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"":"io task","client":"1","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:40553)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.7593726Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"":"io task","client":"0","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:38996)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.759451142Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"":"io task","client":"1","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:34351)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.759489732Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.760039904Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"client":"1","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.760069325Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"client":"1","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.760084675Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"client":"2","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.760098756Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"client":"2","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.760193738Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"client":"0","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.760228628Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"client":"0","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.760683308Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"client":"1","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.760731239Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"client":"2","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.760798071Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"client":"0","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.761298322Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"client":"1","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.761330922Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"client":"1","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.761346443Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.761413514Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"client":"2","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.761447535Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"client":"2","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.761464015Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.761476786Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"client":"0","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.761490436Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"client":"0","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"[0]R flush_numbers[0..12]: [16, 16, 17, 23, 23, 22, 17, 23, 23, 17, 19, 23]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.761504316Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"":"downstairs","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"[0]R generation[0..12]: [8, 8, 9, 11, 11, 10, 9, 11, 11, 9, 10, 11]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.761518296Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"":"downstairs","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.761541917Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"":"downstairs","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"[1]R flush_numbers[0..12]: [16, 16, 17, 23, 23, 22, 17, 23, 23, 17, 19, 23]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.761557567Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"":"downstairs","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"[1]R generation[0..12]: [8, 8, 9, 11, 11, 10, 9, 11, 11, 9, 10, 11]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.761571378Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"":"downstairs","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.761585378Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"":"downstairs","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"[2]R flush_numbers[0..12]: [16, 16, 17, 23, 23, 22, 17, 23, 23, 17, 19, 23]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.761599168Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"":"downstairs","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"[2]R generation[0..12]: [8, 8, 9, 11, 11, 10, 9, 11, 11, 9, 10, 11]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.761612598Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"":"downstairs","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.761629639Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"":"downstairs","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.761643329Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"":"downstairs","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:50:34.76165675Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"":"downstairs","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"Next flush: 24","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.76168835Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"":"downstairs","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.761714791Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"":"downstairs","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.761730411Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.761742921Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"client":"0","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.761756412Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"client":"1","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.761769882Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"client":"2","":"downstairs","session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.761783352Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.761795833Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"dc27fe62-289b-4131-9310-b3d3d1efdd7c is now active with session: 11b201fe-3eac-4cd0-844d-29ba82ff46a3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.761808803Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.761821063Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235,"session_id":"11b201fe-3eac-4cd0-844d-29ba82ff46a3"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:34.761833543Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 399 len 24576 data: 3 2 2 2 2 2 03/30 Write block 573 len 20480 data: 3 3 3 3 2 04/30 Write block 95 len 24576 data: 2 2 2 2 2 2 05/30 Flush 06/30 Write block 324 len 36864 data: 2 2 2 2 2 2 2 2 2 07/30 Read block 361 len 24576 08/30 Read block 319 len 12288 09/30 Write block 451 len 16384 data: 2 2 2 2 10/30 Read block 208 len 40960 11/30 Write block 507 len 12288 data: 4 3 3 12/30 Read block 217 len 4096 13/30 Read block 79 len 28672 14/30 Read block 80 len 28672 15/30 Read block 139 len 16384 16/30 Write block 358 len 16384 data: 3 3 2 2 17/30 Flush 18/30 Write block 438 len 16384 data: 3 3 3 3 19/30 Read block 445 len 24576 20/30 Flush 21/30 Write block 20 len 40960 data: 3 2 2 2 2 2 2 2 2 2 22/30 Write block 236 len 8192 data: 3 3 23/30 Read block 395 len 28672 24/30 Read block 35 len 12288 25/30 Read block 166 len 20480 26/30 Write block 59 len 16384 data: 3 3 2 3 27/30 Read block 473 len 4096 28/30 Write block 40 len 24576 data: 2 2 2 2 3 3 29/30 Write block 123 len 20480 data: 4 3 2 2 2 30/30 Read block 13 len 4096 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 10 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:50:34.780823105Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1235} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 16 16 16 F F F 1 020-039 8 8 8 16 16 16 T T T 2 040-059 9 9 9 17 17 17 T T T 3 060-079 11 11 11 23 23 23 T T T 4 080-099 12 12 12 25 25 25 F F F 5 100-119 12 12 12 25 25 25 F F F 6 120-139 9 9 9 17 17 17 T T T 7 140-159 11 11 11 23 23 23 F F F 8 160-179 11 11 11 23 23 23 F F F 9 180-199 9 9 9 17 17 17 F F F 10 200-219 10 10 10 19 19 19 F F F 11 220-239 11 11 11 23 23 23 T T T 12 240-259 3 3 3 6 6 6 F F F 13 260-279 4 4 4 9 9 9 F F F 14 280-299 11 11 11 23 23 23 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 12 12 12 26 26 26 F F F 17 340-359 12 12 12 26 26 26 F F F 18 360-379 12 12 12 26 26 26 F F F 19 380-399 12 12 12 25 25 25 F F F 20 400-419 12 12 12 25 25 25 F F F 21 420-439 12 12 12 27 27 27 F F F 22 440-459 12 12 12 27 27 27 F F F 23 460-479 8 8 8 16 16 16 F F F 24 480-499 9 9 9 17 17 17 F F F 25 500-519 12 12 12 26 26 26 F F F 26 520-539 11 11 11 23 23 23 F F F 27 540-559 11 11 11 23 23 23 F F F 28 560-579 12 12 12 25 25 25 F F F 29 580-599 5 5 5 12 12 12 F F F Max gen: 12, Max flush: 27 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 16 16 16 F F F 1 020-039 13 13 13 28 28 28 F F F 2 040-059 13 13 13 28 28 28 F F F 3 060-079 13 13 13 28 28 28 F F F 4 080-099 12 12 12 25 25 25 F F F 5 100-119 12 12 12 25 25 25 F F F 6 120-139 13 13 13 28 28 28 F F F 7 140-159 11 11 11 23 23 23 F F F 8 160-179 11 11 11 23 23 23 F F F 9 180-199 9 9 9 17 17 17 F F F 10 200-219 10 10 10 19 19 19 F F F 11 220-239 13 13 13 28 28 28 F F F 12 240-259 3 3 3 6 6 6 F F F 13 260-279 4 4 4 9 9 9 F F F 14 280-299 11 11 11 23 23 23 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 12 12 12 26 26 26 F F F 17 340-359 12 12 12 26 26 26 F F F 18 360-379 12 12 12 26 26 26 F F F 19 380-399 12 12 12 25 25 25 F F F 20 400-419 12 12 12 25 25 25 F F F 21 420-439 12 12 12 27 27 27 F F F 22 440-459 12 12 12 27 27 27 F F F 23 460-479 8 8 8 16 16 16 F F F 24 480-499 9 9 9 17 17 17 F F F 25 500-519 12 12 12 26 26 26 F F F 26 520-539 11 11 11 23 23 23 F F F 27 540-559 11 11 11 23 23 23 F F F 28 560-579 12 12 12 25 25 25 F F F 29 580-599 5 5 5 12 12 12 F F F Max gen: 13, Max flush: 28 Begin loop 7 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.076649524Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:50:39.11980477Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.120198699Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.151986589Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.152589682Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.152624902Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.152639743Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.152654303Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243} {"msg":"Crucible 7df9b0c8-36ce-479b-af45-9358bdbf26ae has session id: 80254d2c-741c-4a98-82ec-13c5e4b1e9f9","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.153094633Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"Upstairs opts: Upstairs UUID: 7df9b0c8-36ce-479b-af45-9358bdbf26ae, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.153128143Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"Crucible stats registered with UUID: 7df9b0c8-36ce-479b-af45-9358bdbf26ae","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.153143083Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.184047644Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.184076245Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243} {"msg":"7df9b0c8-36ce-479b-af45-9358bdbf26ae active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.184142776Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.184163356Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"client":"0","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.184202117Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"client":"0","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.184233648Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"client":"1","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.184264249Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"client":"1","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.184347351Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"client":"2","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.184379411Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"client":"2","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.184519054Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"":"io task","client":"2","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.184601176Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"":"io task","client":"0","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.184641587Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"":"io task","client":"1","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:55413)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.184672558Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"":"io task","client":"2","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:34094)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.184702978Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"":"io task","client":"0","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:36951)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.184729399Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"":"io task","client":"1","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.184984624Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"client":"2","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.185014655Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"client":"2","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.185171778Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"client":"1","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.185201039Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"client":"1","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.185302071Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"client":"0","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.185327842Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"client":"0","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.18573183Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"client":"2","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.185760551Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"client":"1","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.186112449Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"client":"0","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.186249102Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"client":"1","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.186271512Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"client":"1","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.186285842Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.186350084Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"client":"2","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.186372204Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"client":"2","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.186386145Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.186567799Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"client":"0","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.186586569Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"client":"0","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"[0]R flush_numbers[0..12]: [16, 28, 28, 28, 25, 25, 28, 23, 23, 17, 19, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.186601019Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"":"downstairs","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"[0]R generation[0..12]: [8, 13, 13, 13, 12, 12, 13, 11, 11, 9, 10, 13]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.18661533Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"":"downstairs","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.18662973Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"":"downstairs","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"[1]R flush_numbers[0..12]: [16, 28, 28, 28, 25, 25, 28, 23, 23, 17, 19, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.1866434Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"":"downstairs","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"[1]R generation[0..12]: [8, 13, 13, 13, 12, 12, 13, 11, 11, 9, 10, 13]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.18665701Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"":"downstairs","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.186670981Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"":"downstairs","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"[2]R flush_numbers[0..12]: [16, 28, 28, 28, 25, 25, 28, 23, 23, 17, 19, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.186684761Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"":"downstairs","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"[2]R generation[0..12]: [8, 13, 13, 13, 12, 12, 13, 11, 11, 9, 10, 13]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.186698542Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"":"downstairs","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.186712532Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"":"downstairs","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.186725562Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"":"downstairs","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.186742363Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"":"downstairs","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"Next flush: 29","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.186756393Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"":"downstairs","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:50:39.186769673Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"":"downstairs","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.186789173Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.186801544Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"client":"0","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.186814814Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"client":"1","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.186828084Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"client":"2","":"downstairs","session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.186841625Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.186853935Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"7df9b0c8-36ce-479b-af45-9358bdbf26ae is now active with session: 80254d2c-741c-4a98-82ec-13c5e4b1e9f9","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.186866965Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.186879015Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243,"session_id":"80254d2c-741c-4a98-82ec-13c5e4b1e9f9"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:39.186891436Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Write block 408 len 40960 data: 3 3 2 2 2 3 3 3 2 2 04/30 Write block 282 len 24576 data: 2 2 3 3 3 2 05/30 Read block 528 len 20480 06/30 Flush 07/30 Flush 08/30 Write block 461 len 20480 data: 2 2 3 3 3 09/30 Write block 138 len 40960 data: 2 2 2 2 3 3 3 4 3 3 10/30 Write block 54 len 32768 data: 2 2 3 3 3 4 4 3 11/30 Read block 17 len 24576 12/30 Read block 74 len 32768 13/30 Read block 288 len 4096 14/30 Write block 552 len 24576 data: 3 2 2 2 2 2 15/30 Write block 161 len 16384 data: 2 3 3 3 16/30 Read block 72 len 32768 17/30 Read block 379 len 24576 18/30 Read block 152 len 28672 19/30 Write block 451 len 12288 data: 3 3 3 20/30 Write block 192 len 20480 data: 3 3 3 3 2 21/30 Read block 501 len 32768 22/30 Write block 45 len 24576 data: 4 3 3 3 3 3 23/30 Write block 195 len 32768 data: 4 3 4 4 3 3 4 4 24/30 Write block 440 len 8192 data: 4 4 25/30 Write block 325 len 12288 data: 3 3 3 26/30 Read block 233 len 24576 27/30 Write block 350 len 36864 data: 3 3 3 2 3 3 3 3 4 28/30 Read block 68 len 4096 29/30 Read block 435 len 36864 30/30 Write block 202 len 24576 data: 5 4 3 2 2 3 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 5 Done Done Done false 1008 Acked Write 10 Done Done Done false 1009 Acked Write 8 Done Done Done false 1010 Acked Read 6 Done Done Done false 1011 Acked Read 8 Done Done Done false 1012 Acked Read 1 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1006 1006 1006 Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:50:39.20597787Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1243} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 16 16 16 F F F 1 020-039 13 13 13 28 28 28 F F F 2 040-059 13 13 13 28 28 28 T T T 3 060-079 13 13 13 28 28 28 T T T 4 080-099 12 12 12 25 25 25 F F F 5 100-119 12 12 12 25 25 25 F F F 6 120-139 13 13 13 28 28 28 T T T 7 140-159 11 11 11 23 23 23 T T T 8 160-179 11 11 11 23 23 23 T T T 9 180-199 9 9 9 17 17 17 T T T 10 200-219 10 10 10 19 19 19 T T T 11 220-239 13 13 13 28 28 28 F F F 12 240-259 3 3 3 6 6 6 F F F 13 260-279 4 4 4 9 9 9 F F F 14 280-299 14 14 14 31 31 31 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 12 12 12 26 26 26 T T T 17 340-359 12 12 12 26 26 26 T T T 18 360-379 12 12 12 26 26 26 F F F 19 380-399 12 12 12 25 25 25 F F F 20 400-419 14 14 14 31 31 31 F F F 21 420-439 12 12 12 27 27 27 F F F 22 440-459 12 12 12 27 27 27 T T T 23 460-479 8 8 8 16 16 16 T T T 24 480-499 9 9 9 17 17 17 F F F 25 500-519 12 12 12 26 26 26 F F F 26 520-539 11 11 11 23 23 23 F F F 27 540-559 11 11 11 23 23 23 T T T 28 560-579 12 12 12 25 25 25 F F F 29 580-599 5 5 5 12 12 12 F F F Max gen: 14, Max flush: 31 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 16 16 16 F F F 1 020-039 13 13 13 28 28 28 F F F 2 040-059 15 15 15 32 32 32 F F F 3 060-079 15 15 15 32 32 32 F F F 4 080-099 12 12 12 25 25 25 F F F 5 100-119 12 12 12 25 25 25 F F F 6 120-139 15 15 15 32 32 32 F F F 7 140-159 15 15 15 32 32 32 F F F 8 160-179 15 15 15 32 32 32 F F F 9 180-199 15 15 15 32 32 32 F F F 10 200-219 15 15 15 32 32 32 F F F 11 220-239 13 13 13 28 28 28 F F F 12 240-259 3 3 3 6 6 6 F F F 13 260-279 4 4 4 9 9 9 F F F 14 280-299 14 14 14 31 31 31 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 15 15 15 32 32 32 F F F 17 340-359 15 15 15 32 32 32 F F F 18 360-379 12 12 12 26 26 26 F F F 19 380-399 12 12 12 25 25 25 F F F 20 400-419 14 14 14 31 31 31 F F F 21 420-439 12 12 12 27 27 27 F F F 22 440-459 15 15 15 32 32 32 F F F 23 460-479 15 15 15 32 32 32 F F F 24 480-499 9 9 9 17 17 17 F F F 25 500-519 12 12 12 26 26 26 F F F 26 520-539 11 11 11 23 23 23 F F F 27 540-559 15 15 15 32 32 32 F F F 28 560-579 12 12 12 25 25 25 F F F 29 580-599 5 5 5 12 12 12 F F F Max gen: 15, Max flush: 32 Begin loop 8 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.537678928Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.582742125Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.583313687Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.583351458Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.583367048Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.583383019Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251} {"msg":"Crucible bbb52d6f-e9d2-443c-bab0-e402e67eae61 has session id: aa2f5104-18e5-4378-a920-272c5e9de6d3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.583837599Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"Upstairs opts: Upstairs UUID: bbb52d6f-e9d2-443c-bab0-e402e67eae61, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.58388711Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"Crucible stats registered with UUID: bbb52d6f-e9d2-443c-bab0-e402e67eae61","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.58390912Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.614656377Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.614688148Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251} {"msg":"bbb52d6f-e9d2-443c-bab0-e402e67eae61 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.614718569Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.614737149Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"client":"0","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.61476634Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"client":"0","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.61478586Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"client":"1","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.61480449Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"client":"1","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.614823061Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"client":"2","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.614853502Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"client":"2","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.614883972Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"":"io task","client":"2","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.614917383Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"":"io task","client":"0","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.614952674Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"":"io task","client":"1","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:40965)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.615166858Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"":"io task","client":"2","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:49123)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.615213739Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"":"io task","client":"0","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:57098)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.615251Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"":"io task","client":"1","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.615746941Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"client":"2","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.615775282Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"client":"2","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.615790602Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"client":"1","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.615806102Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"client":"1","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.615827483Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"client":"0","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.615853963Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"client":"0","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.616408655Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"client":"2","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.616538708Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"client":"1","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.616578539Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"client":"0","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.616967568Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"client":"2","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.617010958Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"client":"2","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.617041219Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.61706671Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"client":"1","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.61709585Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"client":"1","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.617126981Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.617149001Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"client":"0","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.617172902Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"client":"0","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"[0]R flush_numbers[0..12]: [16, 28, 32, 32, 25, 25, 32, 32, 32, 32, 32, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.617197662Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"":"downstairs","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"[0]R generation[0..12]: [8, 13, 15, 15, 12, 12, 15, 15, 15, 15, 15, 13]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.617223913Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"":"downstairs","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.617249234Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"":"downstairs","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"[1]R flush_numbers[0..12]: [16, 28, 32, 32, 25, 25, 32, 32, 32, 32, 32, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.617272524Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"":"downstairs","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"[1]R generation[0..12]: [8, 13, 15, 15, 12, 12, 15, 15, 15, 15, 15, 13]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.617298115Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"":"downstairs","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.617325785Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"":"downstairs","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"[2]R flush_numbers[0..12]: [16, 28, 32, 32, 25, 25, 32, 32, 32, 32, 32, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.617352266Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"":"downstairs","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"[2]R generation[0..12]: [8, 13, 15, 15, 12, 12, 15, 15, 15, 15, 15, 13]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.617379856Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"":"downstairs","":"downstairs","session_id":Wait for a query_work_queue command to finish before sending IO "aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.617419717Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"":"downstairs","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.617448968Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"":"downstairs","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.617477868Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"":"downstairs","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"Next flush: 33","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.617511059Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"":"downstairs","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.61755535Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"":"downstairs","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"No downstairs reconciliation required"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.617589711Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.617617991Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"client":"0","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.617640822Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"client":"1","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.617654992Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"client":"2","":"downstairs","session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.617668533Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.617680993Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"bbb52d6f-e9d2-443c-bab0-e402e67eae61 is now active with session: aa2f5104-18e5-4378-a920-272c5e9de6d3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.617693823Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.617706004Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251,"session_id":"aa2f5104-18e5-4378-a920-272c5e9de6d3"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:43.617718384Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 575 len 24576 data: 4 4 3 2 2 2 02/30 Write block 324 len 36864 data: 3 4 4 4 3 3 3 3 3 03/30 Read block 164 len 32768 04/30 Write block 56 len 24576 data: 4 4 4 5 5 4 05/30 Read block 148 len 16384 06/30 Read block 357 len 16384 07/30 Read block 416 len 36864 08/30 Flush 09/30 Write block 270 len 32768 data: 4 4 4 4 4 3 3 3 10/30 Write block 438 len 40960 data: 4 4 5 5 2 2 2 2 2 2 11/30 Write block 511 len 16384 data: 3 3 3 3 12/30 Read block 264 len 28672 13/30 Read block 581 len 40960 14/30 Read block 59 len 4096 15/30 Write block 512 len 32768 data: 4 4 4 4 3 4 4 4 16/30 Read block 519 len 4096 17/30 Read block 112 len 12288 18/30 Write block 292 len 12288 data: 2 2 2 19/30 Read block 477 len 40960 20/30 Read block 92 len 32768 21/30 Flush 22/30 Write block 205 len 16384 data: 3 3 4 3 23/30 Read block 518 len 20480 24/30 Write block 486 len 36864 data: 2 2 2 2 3 3 3 3 3 25/30 Write block 182 len 16384 data: 2 3 3 3 26/30 Read block 32 len 8192 27/30 Read block 591 len 28672 28/30 Write block 187 len 40960 data: 4 4 4 3 3 4 4 4 5 4 29/30 Write block 387 len 40960 data: 2 2 2 2 2 2 2 2 2 2 30/30 Read block 133 len 28672 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 4 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 10 Done Done Done false 1029 Acked Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 9 9 8 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:50:43.638389282Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1251} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 16 16 16 F F F 1 020-039 13 13 13 28 28 28 F F F 2 040-059 16 16 16 33 33 33 F F F 3 060-079 16 16 16 33 33 33 F F F 4 080-099 12 12 12 25 25 25 F F F 5 100-119 12 12 12 25 25 25 F F F 6 120-139 15 15 15 32 32 32 F F F 7 140-159 15 15 15 32 32 32 F F F 8 160-179 15 15 15 32 32 32 F F F 9 180-199 15 15 15 32 32 32 T T T 10 200-219 15 15 15 32 32 32 T T T 11 220-239 13 13 13 28 28 28 F F F 12 240-259 3 3 3 6 6 6 F F F 13 260-279 16 16 16 34 34 34 F F F 14 280-299 16 16 16 34 34 34 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 16 16 16 33 33 33 F F F 17 340-359 15 15 15 32 32 32 F F F 18 360-379 12 12 12 26 26 26 F F F 19 380-399 12 12 12 25 25 25 T T T 20 400-419 14 14 14 31 31 31 F F F 21 420-439 16 16 16 34 34 34 F F F 22 440-459 16 16 16 34 34 34 F F F 23 460-479 15 15 15 32 32 32 F F F 24 480-499 9 9 9 17 17 17 T T T 25 500-519 16 16 16 34 34 34 F F F 26 520-539 11 11 11 23 23 23 F F F 27 540-559 15 15 15 32 32 32 F F F 28 560-579 16 16 16 33 33 33 F F F 29 580-599 16 16 16 33 33 33 F F F Max gen: 16, Max flush: 34 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 16 16 16 F F F 1 020-039 13 13 13 28 28 28 F F F 2 040-059 16 16 16 33 33 33 F F F 3 060-079 16 16 16 33 33 33 F F F 4 080-099 12 12 12 25 25 25 F F F 5 100-119 12 12 12 25 25 25 F F F 6 120-139 15 15 15 32 32 32 F F F 7 140-159 15 15 15 32 32 32 F F F 8 160-179 15 15 15 32 32 32 F F F 9 180-199 17 17 17 35 35 35 F F F 10 200-219 17 17 17 35 35 35 F F F 11 220-239 13 13 13 28 28 28 F F F 12 240-259 3 3 3 6 6 6 F F F 13 260-279 16 16 16 34 34 34 F F F 14 280-299 16 16 16 34 34 34 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 16 16 16 33 33 33 F F F 17 340-359 15 15 15 32 32 32 F F F 18 360-379 12 12 12 26 26 26 F F F 19 380-399 17 17 17 35 35 35 F F F 20 400-419 14 14 14 31 31 31 F F F 21 420-439 16 16 16 34 34 34 F F F 22 440-459 16 16 16 34 34 34 F F F 23 460-479 15 15 15 32 32 32 F F F 24 480-499 17 17 17 35 35 35 F F F 25 500-519 16 16 16 34 34 34 F F F 26 520-539 11 11 11 23 23 23 F F F 27 540-559 15 15 15 32 32 32 F F F 28 560-579 16 16 16 33 33 33 F F F 29 580-599 16 16 16 33 33 33 F F F Max gen: 17, Max flush: 35 Begin loop 9 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.895524902Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.939006165Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.939584358Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.939620648Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.939634929Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.939649099Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259} {"msg":"Crucible 70f90c09-46f4-4b8d-9aef-a6950b35c2f1 has session id: 88bf2a28-f076-45a6-8ba2-3cf9d5969d5b","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.940047098Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"Upstairs opts: Upstairs UUID: 70f90c09-46f4-4b8d-9aef-a6950b35c2f1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.940094949Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"Crucible stats registered with UUID: 70f90c09-46f4-4b8d-9aef-a6950b35c2f1","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.940121709Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.971684294Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.971723445Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259} {"msg":"70f90c09-46f4-4b8d-9aef-a6950b35c2f1 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.971752196Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.971770736Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"client":"0","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.971800537Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"client":"0","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.971819937Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"client":"1","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.971838207Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"client":"1","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.971856198Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"client":"2","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.971874648Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"client":"2","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.971892649Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"":"io task","client":"2","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.971911489Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"":"io task","client":"1","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.97193024Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"":"io task","client":"0","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"ds_connection connected from Ok(127.0.0.1:40690)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.972004411Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"":"io task","client":"2","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"ds_connection connected from Ok(127.0.0.1:34344)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.972214235Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"":"io task","client":"1","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"ds_connection connected from Ok(127.0.0.1:44470)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.972254967Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"":"io task","client":"0","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.972675576Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"client":"0","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.972717746Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"client":"0","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.972754967Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"client":"1","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.972832549Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"client":"1","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.97286399Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"client":"2","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.97289666Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"client":"2","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.97334554Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"client":"0","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.973374411Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"client":"1","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.973669957Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"client":"2","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.973873891Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"client":"0","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.973900692Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"client":"0","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.973915303Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.973977114Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"client":"1","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.973996134Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"client":"1","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.974009825Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.974152528Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"client":"2","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.974170758Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"client":"2","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"[0]R flush_numbers[0..12]: [16, 28, 33, 33, 25, 25, 32, 32, 32, 35, 35, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.974187738Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"":"downstairs","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"[0]R generation[0..12]: [8, 13, 16, 16, 12, 12, 15, 15, 15, 17, 17, 13]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.974202049Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"":"downstairs","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.974216159Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"":"downstairs","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"[1]R flush_numbers[0..12]: [16, 28, 33, 33, 25, 25, 32, 32, 32, 35, 35, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.974229969Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"":"downstairs","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"[1]R generation[0..12]: [8, 13, 16, 16, 12, 12, 15, 15, 15, 17, 17, 13]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.97424362Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"":"downstairs","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.97425776Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"":"downstairs","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"[2]R flush_numbers[0..12]: [16, 28, 33, 33, 25, 25, 32, 32, 32, 35, 35, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.97427149Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"":"downstairs","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"[2]R generation[0..12]: [8, 13, 16, 16, 12, 12, 15, 15, 15, 17, 17, 13]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.974285331Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"":"downstairs","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.974299211Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"":"downstairs","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.974312491Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"":"downstairs","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.974325761Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"":"downstairs","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"Next flush: 36","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.974338742Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"":"downstairs","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:50:47.974351712Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"":"downstairs","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.974435274Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.974470404Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"client":"0","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.974503775Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"client":"1","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.974533376Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"client":"2","":"downstairs","session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.974562797Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.974590417Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"70f90c09-46f4-4b8d-9aef-a6950b35c2f1 is now active with session: 88bf2a28-f076-45a6-8ba2-3cf9d5969d5b","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.974618378Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-28T04:50:47.974645768Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259,"session_id":"88bf2a28-f076-45a6-8ba2-3cf9d5969d5b"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:47.974674139Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 23 len 8192 data: 3 3 02/30 Write block 441 len 4096 data: 6 03/30 Write block 569 len 28672 data: 3 3 3 3 4 4 5 04/30 Write block 95 len 8192 data: 3 3 05/30 Read block 319 len 8192 06/30 Write block 454 len 28672 data: 3 2 2 2 2 2 2 07/30 Flush 08/30 Write block 134 len 32768 data: 3 3 3 2 3 3 3 3 09/30 Write block 243 len 32768 data: 2 2 2 2 2 2 2 2 10/30 Flush 11/30 Write block 49 len 12288 data: 4 4 3 12/30 Write block 13 len 36864 data: 2 2 3 3 3 3 4 4 3 13/30 Write block 121 len 12288 data: 4 4 5 14/30 Read block 255 len 4096 15/30 Write block 245 len 36864 data: 3 3 3 3 3 3 2 2 2 16/30 Write block 269 len 24576 data: 4 5 5 5 5 5 17/30 Flush 18/30 Write block 92 len 4096 data: 3 19/30 Read block 522 len 20480 20/30 Write block 526 len 8192 data: 3 3 21/30 Flush 22/30 Write block 64 len 20480 data: 3 3 3 4 4 23/30 Read block 115 len 40960 24/30 Read block 65 len 8192 25/30 Write block 51 len 32768 data: 4 3 2 3 3 5 5 5 26/30 Read block 244 len 24576 27/30 Read block 426 len 12288 28/30 Read block 297 len 16384 29/30 Write block 439 len 24576 data: 5 6 7 3 3 3 30/30 Write block 251 len 36864 data: 3 3 3 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 5 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 6 Sent Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 1 1 4 Done 7 8 8 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:50:47.992733641Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1259} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 38 38 38 F F F 1 020-039 18 18 18 38 38 38 F F F 2 040-059 18 18 18 38 38 38 T T T 3 060-079 16 16 16 33 33 33 T T T 4 080-099 18 18 18 39 39 39 F F F 5 100-119 12 12 12 25 25 25 F F F 6 120-139 18 18 18 38 38 38 F F F 7 140-159 18 18 18 37 37 37 F F F 8 160-179 15 15 15 32 32 32 F F F 9 180-199 17 17 17 35 35 35 F F F 10 200-219 17 17 17 35 35 35 F F F 11 220-239 13 13 13 28 28 28 F F F 12 240-259 18 18 18 38 38 38 T T T 13 260-279 18 18 18 38 38 38 F F F 14 280-299 16 16 16 34 34 34 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 16 16 16 33 33 33 F F F 17 340-359 15 15 15 32 32 32 F F F 18 360-379 12 12 12 26 26 26 F F F 19 380-399 17 17 17 35 35 35 F F F 20 400-419 14 14 14 31 31 31 F F F 21 420-439 16 16 16 34 34 34 T T T 22 440-459 18 18 18 36 36 36 T T T 23 460-479 18 18 18 36 36 36 F F F 24 480-499 17 17 17 35 35 35 F F F 25 500-519 16 16 16 34 34 34 F F F 26 520-539 18 18 18 39 39 39 F F F 27 540-559 15 15 15 32 32 32 F F F 28 560-579 18 18 18 36 36 36 F F F 29 580-599 16 16 16 33 33 33 F F F Max gen: 18, Max flush: 39 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 38 38 38 F F F 1 020-039 18 18 18 38 38 38 F F F 2 040-059 19 19 19 40 40 40 F F F 3 060-079 19 19 19 40 40 40 F F F 4 080-099 18 18 18 39 39 39 F F F 5 100-119 12 12 12 25 25 25 F F F 6 120-139 18 18 18 38 38 38 F F F 7 140-159 18 18 18 37 37 37 F F F 8 160-179 15 15 15 32 32 32 F F F 9 180-199 17 17 17 35 35 35 F F F 10 200-219 17 17 17 35 35 35 F F F 11 220-239 13 13 13 28 28 28 F F F 12 240-259 19 19 19 40 40 40 F F F 13 260-279 18 18 18 38 38 38 F F F 14 280-299 16 16 16 34 34 34 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 16 16 16 33 33 33 F F F 17 340-359 15 15 15 32 32 32 F F F 18 360-379 12 12 12 26 26 26 F F F 19 380-399 17 17 17 35 35 35 F F F 20 400-419 14 14 14 31 31 31 F F F 21 420-439 19 19 19 40 40 40 F F F 22 440-459 19 19 19 40 40 40 F F F 23 460-479 18 18 18 36 36 36 F F F 24 480-499 17 17 17 35 35 35 F F F 25 500-519 16 16 16 34 34 34 F F F 26 520-539 18 18 18 39 39 39 F F F 27 540-559 15 15 15 32 32 32 F F F 28 560-579 18 18 18 36 36 36 F F F 29 580-599 16 16 16 33 33 33 F F F Max gen: 19, Max flush: 40 Begin loop 10 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.258883457Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.302508483Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.303121616Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.303156977Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.303172537Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.303187438Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267} {"msg":"Crucible e66b13eb-d9b6-4d5d-abcd-ac9e418959d6 has session id: 3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.303697729Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"Upstairs opts: Upstairs UUID: e66b13eb-d9b6-4d5d-abcd-ac9e418959d6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.3037406Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"Crucible stats registered with UUID: e66b13eb-d9b6-4d5d-abcd-ac9e418959d6","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.30376568Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.335378216Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.335424957Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267} {"msg":"e66b13eb-d9b6-4d5d-abcd-ac9e418959d6 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.335487819Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.335515889Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"client":"0","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.335541429Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"client":"0","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.33556088Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"client":"1","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.33558169Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"client":"1","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.335609181Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"client":"2","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.335636642Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"client":"2","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.335662282Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"":"io task","client":"2","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.335689493Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"":"io task","client":"0","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.335709893Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"":"io task","client":"1","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"ds_connection connected from Ok(127.0.0.1:48688)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.335737834Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"":"io task","client":"2","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"ds_connection connected from Ok(127.0.0.1:39782)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.335765985Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"":"io task","client":"0","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"ds_connection connected from Ok(127.0.0.1:60581)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.335804065Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"":"io task","client":"1","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.336551031Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"client":"2","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.336577872Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"client":"2","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.336592612Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"client":"0","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.336606283Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"client":"0","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.336731835Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"client":"1","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.336755476Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"client":"1","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.337312068Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"client":"2","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.337344259Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"client":"1","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.337361979Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"client":"0","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.337915781Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"client":"1","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.337945532Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"client":"1","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.337961972Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.337976732Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"client":"2","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.337992233Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"client":"2","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.338007003Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.338020523Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"client":"0","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.338035394Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"client":"0","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"[0]R flush_numbers[0..12]: [38, 38, 40, 40, 39, 25, 38, 37, 32, 35, 35, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.338050594Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"":"downstairs","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"[0]R generation[0..12]: [18, 18, 19, 19, 18, 12, 18, 18, 15, 17, 17, 13]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.338065844Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"":"downstairs","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.338081265Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"":"downstairs","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"[1]R flush_numbers[0..12]: [38, 38, 40, 40, 39, 25, 38, 37, 32, 35, 35, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.338096375Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"":"downstairs","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"[1]R generation[0..12]: [18, 18, 19, 19, 18, 12, 18, 18, 15, 17, 17, 13]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.338114805Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"":"downstairs","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.338130366Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"":"downstairs","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"[2]R flush_numbers[0..12]: [38, 38, 40, 40, 39, 25, 38, 37, 32, 35, 35, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.338145506Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"":"downstairs","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"[2]R generation[0..12]: [18, 18, 19, 19, 18, 12, 18, 18, 15, 17, 17, 13]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.338160736Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"":"downstairs","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.338176147Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"":"downstairs","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:50:52.338190487Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"":"downstairs","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.338211987Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"":"downstairs","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"Next flush: 41","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.338226568Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"":"downstairs","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.338241178Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"":"downstairs","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.338255608Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.338269059Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"client":"0","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.338283789Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"client":"1","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.338298769Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"client":"2","":"downstairs","session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.33831343Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.33832908Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"e66b13eb-d9b6-4d5d-abcd-ac9e418959d6 is now active with session: 3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.33834359Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:52.338357411Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267,"session_id":"3d82b67b-2e88-4e9a-bb1b-9ae4ef91ad84"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-28T04:50:52.338371451Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 210 len 16384 data: 4 4 5 5 02/30 Read block 233 len 20480 03/30 Write block 526 len 12288 data: 4 4 3 04/30 Write block 431 len 32768 data: 2 2 2 2 3 3 3 5 05/30 Write block 54 len 32768 data: 4 4 6 6 6 6 6 5 06/30 Read block 298 len 40960 07/30 Read block 76 len 28672 08/30 Write block 508 len 28672 data: 4 4 3 4 5 5 5 09/30 Flush 10/30 Write block 19 len 36864 data: 5 5 4 3 4 4 3 3 3 11/30 Flush 12/30 Write block 233 len 36864 data: 2 3 3 4 4 3 3 3 3 13/30 Flush 14/30 Write block 101 len 16384 data: 2 3 3 3 15/30 Read block 234 len 32768 16/30 Write block 421 len 32768 data: 2 2 2 2 2 2 2 2 17/30 Write block 405 len 8192 data: 3 3 18/30 Write block 186 len 24576 data: 5 5 5 5 4 4 19/30 Write block 395 len 40960 data: 3 3 2 3 4 3 3 3 3 3 20/30 Flush 21/30 Write block 200 len 4096 data: 4 22/30 Write block 282 len 4096 data: 3 23/30 Read block 453 len 28672 24/30 Read block 422 len 32768 25/30 Read block 237 len 36864 26/30 Read block 146 len 40960 27/30 Write block 113 len 8192 data: 3 3 28/30 Read block 220 len 32768 29/30 Read block 36 len 8192 30/30 Write block 68 len 36864 data: 5 6 6 6 5 5 5 5 3 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 1 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:50:52.35815578Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1267} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 42 42 42 F F F 1 020-039 20 20 20 42 42 42 F F F 2 040-059 20 20 20 41 41 41 F F F 3 060-079 20 20 20 41 41 41 T T T 4 080-099 18 18 18 39 39 39 F F F 5 100-119 20 20 20 44 44 44 T T T 6 120-139 18 18 18 38 38 38 F F F 7 140-159 18 18 18 37 37 37 F F F 8 160-179 15 15 15 32 32 32 F F F 9 180-199 20 20 20 44 44 44 F F F 10 200-219 20 20 20 41 41 41 T T T 11 220-239 20 20 20 43 43 43 F F F 12 240-259 20 20 20 43 43 43 F F F 13 260-279 18 18 18 38 38 38 F F F 14 280-299 16 16 16 34 34 34 T T T 15 300-319 2 2 2 3 3 3 F F F 16 320-339 16 16 16 33 33 33 F F F 17 340-359 15 15 15 32 32 32 F F F 18 360-379 12 12 12 26 26 26 F F F 19 380-399 20 20 20 44 44 44 F F F 20 400-419 20 20 20 44 44 44 F F F 21 420-439 20 20 20 44 44 44 F F F 22 440-459 19 19 19 40 40 40 F F F 23 460-479 18 18 18 36 36 36 F F F 24 480-499 17 17 17 35 35 35 F F F 25 500-519 20 20 20 41 41 41 F F F 26 520-539 20 20 20 41 41 41 F F F 27 540-559 15 15 15 32 32 32 F F F 28 560-579 18 18 18 36 36 36 F F F 29 580-599 16 16 16 33 33 33 F F F Max gen: 20, Max flush: 44 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 42 42 42 F F F 1 020-039 20 20 20 42 42 42 F F F 2 040-059 20 20 20 41 41 41 F F F 3 060-079 21 21 21 45 45 45 F F F 4 080-099 18 18 18 39 39 39 F F F 5 100-119 21 21 21 45 45 45 F F F 6 120-139 18 18 18 38 38 38 F F F 7 140-159 18 18 18 37 37 37 F F F 8 160-179 15 15 15 32 32 32 F F F 9 180-199 20 20 20 44 44 44 F F F 10 200-219 21 21 21 45 45 45 F F F 11 220-239 20 20 20 43 43 43 F F F 12 240-259 20 20 20 43 43 43 F F F 13 260-279 18 18 18 38 38 38 F F F 14 280-299 21 21 21 45 45 45 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 16 16 16 33 33 33 F F F 17 340-359 15 15 15 32 32 32 F F F 18 360-379 12 12 12 26 26 26 F F F 19 380-399 20 20 20 44 44 44 F F F 20 400-419 20 20 20 44 44 44 F F F 21 420-439 20 20 20 44 44 44 F F F 22 440-459 19 19 19 40 40 40 F F F 23 460-479 18 18 18 36 36 36 F F F 24 480-499 17 17 17 35 35 35 F F F 25 500-519 20 20 20 41 41 41 F F F 26 520-539 20 20 20 41 41 41 F F F 27 540-559 15 15 15 32 32 32 F F F 28 560-579 18 18 18 36 36 36 F F F 29 580-599 16 16 16 33 33 33 F F F Max gen: 21, Max flush: 45 Begin loop 11 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.616535997Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:50:56.658868556Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.659225633Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.690783708Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.69134661Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.691382641Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.691397281Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.691413632Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275} {"msg":"Crucible 4da123b9-7bcc-4a5d-b9e0-e47972960ddc has session id: a870aac8-6fda-4f4d-83ea-d5e37c944631","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.69181629Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"Upstairs opts: Upstairs UUID: 4da123b9-7bcc-4a5d-b9e0-e47972960ddc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.691863451Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"Crucible stats registered with UUID: 4da123b9-7bcc-4a5d-b9e0-e47972960ddc","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.691886452Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.722406494Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.722433734Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275} {"msg":"4da123b9-7bcc-4a5d-b9e0-e47972960ddc active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.722503576Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.722530237Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"client":"0","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.722554567Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"client":"0","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.722569388Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"client":"1","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.722586428Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"client":"1","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.722615178Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"client":"2","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.722637399Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"client":"2","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.72266175Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"":"io task","client":"2","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.7226898Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"":"io task","client":"0","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.722716811Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"":"io task","client":"1","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"ds_connection connected from Ok(127.0.0.1:36498)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.722743491Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"":"io task","client":"1","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"ds_connection connected from Ok(127.0.0.1:57269)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.722772812Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"":"io task","client":"2","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"ds_connection connected from Ok(127.0.0.1:64054)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.722807323Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"":"io task","client":"0","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.723435096Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"client":"2","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.723463087Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"client":"2","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.7236213Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"client":"1","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.723660061Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"client":"1","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.723689882Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"client":"0","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.723719053Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"client":"0","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.724182812Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"client":"1","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.724229864Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"client":"2","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.724264524Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"client":"0","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.724742185Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"client":"2","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.724767495Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"client":"2","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.724781585Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.724863037Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"client":"1","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.724897288Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"client":"1","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.724919088Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.724932139Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"client":"0","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.724945489Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"client":"0","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"[0]R flush_numbers[0..12]: [42, 42, 41, 45, 39, 45, 38, 37, 32, 44, 45, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.724959359Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"":"downstairs","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"[0]R generation[0..12]: [20, 20, 20, 21, 18, 21, 18, 18, 15, 20, 21, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.72497314Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"":"downstairs","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.72498733Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"":"downstairs","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"[1]R flush_numbers[0..12]: [42, 42, 41, 45, 39, 45, 38, 37, 32, 44, 45, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.72500097Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"":"downstairs","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"[1]R generation[0..12]: [20, 20, 20, 21, 18, 21, 18, 18, 15, 20, 21, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.725015341Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"":"downstairs","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.725033511Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"":"downstairs","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"[2]R flush_numbers[0..12]: [42, 42, 41, 45, 39, 45, 38, 37, 32, 44, 45, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.725048571Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"":"downstairs","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"[2]R generation[0..12]: [20, 20, 20, 21, 18, 21, 18, 18, 15, 20, 21, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.725062262Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"":"downstairs","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:50:56.725076202Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"":"downstairs","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.725102972Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"":"downstairs","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.725129093Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"":"downstairs","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"Next flush: 46","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.725151384Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"":"downstairs","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.725176904Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"":"downstairs","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-28T04:50:56.725201475Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.725224815Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"client":"0","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.725249146Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"client":"1","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.725265436Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"client":"2","":"downstairs","session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.725279146Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.725291767Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"4da123b9-7bcc-4a5d-b9e0-e47972960ddc is now active with session: a870aac8-6fda-4f4d-83ea-d5e37c944631","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.725304687Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.725320137Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275,"session_id":"a870aac8-6fda-4f4d-83ea-d5e37c944631"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:50:56.725332957Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 597 len 4096 data: 3 02/30 Read block 171 len 32768 03/30 Write block 19 len 28672 data: 6 6 5 4 5 5 4 04/30 Read block 168 len 12288 05/30 Flush 06/30 Write block 11 len 4096 data: 2 07/30 Read block 106 len 36864 08/30 Read block 55 len 28672 09/30 Read block 543 len 16384 10/30 Write block 89 len 4096 data: 3 11/30 Write block 89 len 40960 data: 4 3 3 4 3 3 4 4 3 3 12/30 Read block 396 len 8192 13/30 Read block 435 len 28672 14/30 Write block 422 len 40960 data: 3 3 3 3 3 3 3 2 2 3 15/30 Write block 161 len 12288 data: 3 4 4 16/30 Write block 474 len 12288 data: 2 2 2 17/30 Write block 567 len 16384 data: 2 2 4 4 18/30 Write block 81 len 8192 data: 2 2 19/30 Read block 514 len 40960 20/30 Read block 179 len 36864 21/30 Write block 11 len 28672 data: 3 2 3 3 4 4 4 22/30 Read block 88 len 16384 23/30 Flush 24/30 Flush 25/30 Write block 77 len 4096 data: 3 26/30 Flush 27/30 Write block 590 len 16384 data: 2 2 2 2 28/30 Read block 132 len 24576 29/30 Read block 138 len 8192 30/30 Read block 579 len 28672 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 4 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 7 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:50:56.743770728Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1275} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 22 22 22 47 47 47 F F F 1 020-039 22 22 22 46 46 46 F F F 2 040-059 20 20 20 41 41 41 F F F 3 060-079 22 22 22 49 49 49 F F F 4 080-099 22 22 22 47 47 47 F F F 5 100-119 21 21 21 45 45 45 F F F 6 120-139 18 18 18 38 38 38 F F F 7 140-159 18 18 18 37 37 37 F F F 8 160-179 22 22 22 47 47 47 F F F 9 180-199 20 20 20 44 44 44 F F F 10 200-219 21 21 21 45 45 45 F F F 11 220-239 20 20 20 43 43 43 F F F 12 240-259 20 20 20 43 43 43 F F F 13 260-279 18 18 18 38 38 38 F F F 14 280-299 21 21 21 45 45 45 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 16 16 16 33 33 33 F F F 17 340-359 15 15 15 32 32 32 F F F 18 360-379 12 12 12 26 26 26 F F F 19 380-399 20 20 20 44 44 44 F F F 20 400-419 20 20 20 44 44 44 F F F 21 420-439 22 22 22 47 47 47 F F F 22 440-459 19 19 19 40 40 40 F F F 23 460-479 22 22 22 47 47 47 F F F 24 480-499 17 17 17 35 35 35 F F F 25 500-519 20 20 20 41 41 41 F F F 26 520-539 20 20 20 41 41 41 F F F 27 540-559 15 15 15 32 32 32 F F F 28 560-579 22 22 22 47 47 47 F F F 29 580-599 22 22 22 46 46 46 T T T Max gen: 22, Max flush: 49 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 22 22 22 47 47 47 F F F 1 020-039 22 22 22 46 46 46 F F F 2 040-059 20 20 20 41 41 41 F F F 3 060-079 22 22 22 49 49 49 F F F 4 080-099 22 22 22 47 47 47 F F F 5 100-119 21 21 21 45 45 45 F F F 6 120-139 18 18 18 38 38 38 F F F 7 140-159 18 18 18 37 37 37 F F F 8 160-179 22 22 22 47 47 47 F F F 9 180-199 20 20 20 44 44 44 F F F 10 200-219 21 21 21 45 45 45 F F F 11 220-239 20 20 20 43 43 43 F F F 12 240-259 20 20 20 43 43 43 F F F 13 260-279 18 18 18 38 38 38 F F F 14 280-299 21 21 21 45 45 45 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 16 16 16 33 33 33 F F F 17 340-359 15 15 15 32 32 32 F F F 18 360-379 12 12 12 26 26 26 F F F 19 380-399 20 20 20 44 44 44 F F F 20 400-419 20 20 20 44 44 44 F F F 21 420-439 22 22 22 47 47 47 F F F 22 440-459 19 19 19 40 40 40 F F F 23 460-479 22 22 22 47 47 47 F F F 24 480-499 17 17 17 35 35 35 F F F 25 500-519 20 20 20 41 41 41 F F F 26 520-539 20 20 20 41 41 41 F F F 27 540-559 15 15 15 32 32 32 F F F 28 560-579 22 22 22 47 47 47 F F F 29 580-599 23 23 23 50 50 50 F F F Max gen: 23, Max flush: 50 Begin loop 12 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.016951115Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:51:01.059473868Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.059842416Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.091603475Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.092145146Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.092178597Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.092197508Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.092212498Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283} {"msg":"Crucible 2d9ca40c-0c8c-4f2d-8962-f953f2287141 has session id: c4dffa15-4d97-4728-8935-53daa4bf97b4","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.092682798Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"Upstairs opts: Upstairs UUID: 2d9ca40c-0c8c-4f2d-8962-f953f2287141, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.092728749Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"Crucible stats registered with UUID: 2d9ca40c-0c8c-4f2d-8962-f953f2287141","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.09275184Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.123491206Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.123524997Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283} {"msg":"2d9ca40c-0c8c-4f2d-8962-f953f2287141 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.123558708Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.123575778Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"client":"0","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.123604949Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"client":"0","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.123622859Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"client":"1","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.12364Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"client":"1","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.12366846Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"client":"2","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.123691431Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"client":"2","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.123721791Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"":"io task","client":"2","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.123753432Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"":"io task","client":"0","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.123792333Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"":"io task","client":"1","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:61207)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.123829084Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"":"io task","client":"2","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:62024)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.123916546Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"":"io task","client":"0","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:34730)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.123949967Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"":"io task","client":"1","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.124609711Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"client":"2","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.124640211Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"client":"2","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.124668172Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"client":"1","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.124697853Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"client":"1","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.124763304Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"client":"0","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.124797125Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"client":"0","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.125332766Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"client":"1","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.125363347Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"client":"2","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.125380227Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"client":"0","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.125902039Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"client":"1","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.125932579Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"client":"1","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.12595301Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.12596708Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"client":"2","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.125981731Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"client":"2","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.125996301Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.126010251Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"client":"0","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.126024901Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"client":"0","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"[0]R flush_numbers[0..12]: [47, 46, 41, 49, 47, 45, 38, 37, 47, 44, 45, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.126039682Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"":"downstairs","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"[0]R generation[0..12]: [22, 22, 20, 22, 22, 21, 18, 18, 22, 20, 21, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.126054792Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"":"downstairs","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.126070233Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"":"downstairs","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"[1]R flush_numbers[0..12]: [47, 46, 41, 49, 47, 45, 38, 37, 47, 44, 45, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.126085223Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"":"downstairs","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"[1]R generation[0..12]: [22, 22, 20, 22, 22, 21, 18, 18, 22, 20, 21, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.126100253Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"":"downstairs","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.126115494Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"":"downstairs","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"[2]R flush_numbers[0..12]: [47, 46, 41, 49, 47, 45, 38, 37, 47, 44, 45, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.126130614Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"":"downstairs","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"[2]R generation[0..12]: [22, 22, 20, 22, 22, 21, 18, 18, 22, 20, 21, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.126145604Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"":"downstairs","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.126163204Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"":"downstairs","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.126177835Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"":"downstairs","":"downstairsWait for a query_work_queue command to finish before sending IO ","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.126207745Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"":"downstairs","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"Next flush: 51","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.126234806Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"":"downstairs","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.126263177Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"":"downstairs","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.126291717Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.126318158Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"client":"0","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.126347268Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"client":"1","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.126376879Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"client":"2","":"downstairs","session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.12640681Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","levelDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":30,"time":"2024-11-28T04:51:01.126438181Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"2d9ca40c-0c8c-4f2d-8962-f953f2287141 is now active with session: c4dffa15-4d97-4728-8935-53daa4bf97b4","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.126465451Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.126484802Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283,"session_id":"c4dffa15-4d97-4728-8935-53daa4bf97b4"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:01.126499482Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 468 len 32768 data: 3 3 3 2 2 2 3 3 03/30 Flush 04/30 Flush 05/30 Write block 5 len 16384 data: 2 2 2 2 06/30 Write block 494 len 32768 data: 4 3 2 2 2 2 3 2 07/30 Read block 296 len 12288 08/30 Read block 30 len 4096 09/30 Write block 78 len 16384 data: 2 2 2 3 10/30 Write block 431 len 4096 data: 4 11/30 Read block 5 len 8192 12/30 Read block 255 len 12288 13/30 Read block 2 len 36864 14/30 Write block 222 len 4096 data: 4 15/30 Write block 146 len 16384 data: 4 4 3 3 16/30 Read block 389 len 16384 17/30 Write block 505 len 24576 data: 3 4 5 5 5 4 18/30 Read block 108 len 8192 19/30 Write block 237 len 4096 data: 5 20/30 Read block 397 len 24576 21/30 Read block 450 len 4096 22/30 Write block 174 len 8192 data: 3 3 23/30 Read block 506 len 4096 24/30 Write block 370 len 12288 data: 3 4 4 25/30 Write block 157 len 16384 data: 2 2 2 2 26/30 Write block 196 len 32768 data: 5 5 5 4 5 5 6 5 27/30 Write block 269 len 8192 data: 5 6 28/30 Write block 52 len 8192 data: 4 3 29/30 Read block 211 len 8192 30/30 Write block 240 len 16384 data: 4 4 2 3 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 4 Done Done Done false 1005 Acked Write 8 Done Done Done false 1006 Acked Read 3 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Write 4 Done Done Done false 1009 Acked Write 1 Done Done Done false 1010 Acked Read 2 Done Done Done false 1011 Acked Read 3 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Write 1 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 25 25 25 75 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:51:01.142983649Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1283} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 22 22 22 47 47 47 T T T 1 020-039 22 22 22 46 46 46 F F F 2 040-059 20 20 20 41 41 41 T T T 3 060-079 22 22 22 49 49 49 T T T 4 080-099 22 22 22 47 47 47 T T T 5 100-119 21 21 21 45 45 45 F F F 6 120-139 18 18 18 38 38 38 F F F 7 140-159 18 18 18 37 37 37 T T T 8 160-179 22 22 22 47 47 47 T T T 9 180-199 20 20 20 44 44 44 T T T 10 200-219 21 21 21 45 45 45 T T T 11 220-239 20 20 20 43 43 43 T T T 12 240-259 20 20 20 43 43 43 T T T 13 260-279 18 18 18 38 38 38 T T T 14 280-299 21 21 21 45 45 45 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 16 16 16 33 33 33 F F F 17 340-359 15 15 15 32 32 32 F F F 18 360-379 12 12 12 26 26 26 T T T 19 380-399 20 20 20 44 44 44 F F F 20 400-419 20 20 20 44 44 44 F F F 21 420-439 22 22 22 47 47 47 T T T 22 440-459 19 19 19 40 40 40 F F F 23 460-479 24 24 24 52 52 52 F F F 24 480-499 17 17 17 35 35 35 T T T 25 500-519 20 20 20 41 41 41 T T T 26 520-539 20 20 20 41 41 41 F F F 27 540-559 15 15 15 32 32 32 F F F 28 560-579 22 22 22 47 47 47 F F F 29 580-599 23 23 23 50 50 50 F F F Max gen: 24, Max flush: 52 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 53 53 53 F F F 1 020-039 22 22 22 46 46 46 F F F 2 040-059 25 25 25 53 53 53 F F F 3 060-079 25 25 25 53 53 53 F F F 4 080-099 25 25 25 53 53 53 F F F 5 100-119 21 21 21 45 45 45 F F F 6 120-139 18 18 18 38 38 38 F F F 7 140-159 25 25 25 53 53 53 F F F 8 160-179 25 25 25 53 53 53 F F F 9 180-199 25 25 25 53 53 53 F F F 10 200-219 25 25 25 53 53 53 F F F 11 220-239 25 25 25 53 53 53 F F F 12 240-259 25 25 25 53 53 53 F F F 13 260-279 25 25 25 53 53 53 F F F 14 280-299 21 21 21 45 45 45 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 16 16 16 33 33 33 F F F 17 340-359 15 15 15 32 32 32 F F F 18 360-379 25 25 25 53 53 53 F F F 19 380-399 20 20 20 44 44 44 F F F 20 400-419 20 20 20 44 44 44 F F F 21 420-439 25 25 25 53 53 53 F F F 22 440-459 19 19 19 40 40 40 F F F 23 460-479 24 24 24 52 52 52 F F F 24 480-499 25 25 25 53 53 53 F F F 25 500-519 25 25 25 53 53 53 F F F 26 520-539 20 20 20 41 41 41 F F F 27 540-559 15 15 15 32 32 32 F F F 28 560-579 22 22 22 47 47 47 F F F 29 580-599 23 23 23 50 50 50 F F F Max gen: 25, Max flush: 53 Begin loop 13 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.490263005Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.534834952Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.535564218Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.535605809Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.535621749Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.535640359Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291} {"msg":"Crucible e3fa1c76-2141-4d14-ba63-98d8d75bf780 has session id: ce8dd534-31d9-4b12-a413-3b97be3a9007","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.536023738Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"Upstairs opts: Upstairs UUID: e3fa1c76-2141-4d14-ba63-98d8d75bf780, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.536061039Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"Crucible stats registered with UUID: e3fa1c76-2141-4d14-ba63-98d8d75bf780","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.536076869Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.567058821Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.567090782Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291} {"msg":"e3fa1c76-2141-4d14-ba63-98d8d75bf780 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.567123162Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.567139553Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"client":"0","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.567167833Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"client":"0","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.567185634Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"client":"1","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.567202804Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"client":"1","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.567219724Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"client":"2","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.567247395Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"client":"2","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.567352787Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"":"io task","client":"2","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.567395708Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"":"io task","client":"0","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.567441649Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"":"io task","client":"1","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"ds_connection connected from Ok(127.0.0.1:44136)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.567564432Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"":"io task","client":"2","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"ds_connection connected from Ok(127.0.0.1:46624)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.567601373Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"":"io task","client":"0","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"ds_connection connected from Ok(127.0.0.1:41797)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.567621823Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"":"io task","client":"1","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.568059863Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"client":"2","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.568087383Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"client":"2","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.568239186Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"client":"1","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.568277638Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"client":"1","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.568294248Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"client":"0","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.568308188Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"client":"0","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.568746708Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"client":"2","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.568774648Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"client":"0","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.568920361Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"client":"1","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.569379581Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"client":"0","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.569415172Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"client":"0","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.569437243Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.569452773Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"client":"2","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.569468003Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"client":"2","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.569482733Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.569496664Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"client":"1","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.569511304Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"client":"1","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"[0]R flush_numbers[0..12]: [53, 46, 53, 53, 53, 45, 38, 53, 53, 53, 53, 53]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.569525785Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"":"downstairs","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"[0]R generation[0..12]: [25, 22, 25, 25, 25, 21, 18, 25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.569554265Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"":"downstairs","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.569570655Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"":"downstairs","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"[1]R flush_numbers[0..12]: [53, 46, 53, 53, 53, 45, 38, 53, 53, 53, 53, 53]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.569585476Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"":"downstairs","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"[1]R generation[0..12]: [25, 22, 25, 25, 25, 21, 18, 25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.569600846Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"":"downstairs","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.569616147Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"":"downstairs","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"[2]R flush_numbers[0..12]: [53, 46, 53, 53, 53, 45, 38, 53, 53, 53, 53, 53]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.569631777Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"":"downstairs","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"[2]R generation[0..12]: [25, 22, 25, 25, 25, 21, 18, 25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.569646717Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"":"downstairs","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.569672338Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"":"downstairs","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.569686788Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"":"downstairs","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.569701118Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"":"downstairs","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"Next flush: 54","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.569715489Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"":"downstairs","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.569729759Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"":"downstairs","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.569744219Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.56975767Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"client":"0","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.56977219Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"client":"1","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"Transition from WaitQuorum to Active"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.5697905Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"client":"2","":"downstairs","session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.56980476Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.569818721Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"e3fa1c76-2141-4d14-ba63-98d8d75bf780 is now active with session: ce8dd534-31d9-4b12-a413-3b97be3a9007","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.569833181Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.569846552Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291,"session_id":"ce8dd534-31d9-4b12-a413-3b97be3a9007"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:05.569860042Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 66 len 32768 data: 4 5 6 7 7 7 6 6 02/30 Read block 255 len 4096 03/30 Write block 550 len 40960 data: 3 3 4 3 3 3 3 3 2 2 04/30 Write block 432 len 12288 data: 3 3 3 05/30 Read block 71 len 20480 06/30 Read block 224 len 16384 07/30 Read block 98 len 20480 08/30 Read block 467 len 40960 09/30 Read block 25 len 4096 10/30 Read block 0 len 4096 11/30 Flush 12/30 Write block 352 len 4096 data: 4 13/30 Read block 189 len 24576 14/30 Read block 532 len 36864 15/30 Write block 53 len 36864 data: 4 5 5 7 7 7 7 7 6 16/30 Write block 176 len 36864 data: 3 3 2 2 2 2 3 4 4 17/30 Write block 78 len 36864 data: 3 3 3 4 3 2 2 3 3 18/30 Write block 31 len 24576 data: 2 2 2 2 2 2 19/30 Write block 428 len 40960 data: 4 3 3 5 4 4 4 4 4 4 20/30 Read block 148 len 40960 21/30 Read block 465 len 36864 22/30 Write block 351 len 36864 data: 4 5 3 4 4 4 4 5 4 23/30 Write block 28 len 8192 data: 3 3 24/30 Write block 358 len 28672 data: 6 5 3 3 2 2 2 25/30 Write block 375 len 28672 data: 3 3 4 4 4 4 3 26/30 Write block 330 len 28672 data: 4 4 4 3 3 4 5 27/30 Write block 547 len 40960 data: 3 3 3 4 4 5 4 4 4 4 28/30 Read block 28 len 32768 29/30 Write block 210 len 8192 data: 5 5 30/30 Write block 577 len 12288 data: 4 3 3 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 1 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Write 9 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:51:05.588856064Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1291} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 53 53 53 F F F 1 020-039 22 22 22 46 46 46 T T T 2 040-059 25 25 25 53 53 53 T T T 3 060-079 26 26 26 54 54 54 T T T 4 080-099 25 25 25 53 53 53 T T T 5 100-119 21 21 21 45 45 45 F F F 6 120-139 18 18 18 38 38 38 F F F 7 140-159 25 25 25 53 53 53 F F F 8 160-179 25 25 25 53 53 53 T T T 9 180-199 25 25 25 53 53 53 T T T 10 200-219 25 25 25 53 53 53 T T T 11 220-239 25 25 25 53 53 53 F F F 12 240-259 25 25 25 53 53 53 F F F 13 260-279 25 25 25 53 53 53 F F F 14 280-299 21 21 21 45 45 45 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 16 16 16 33 33 33 T T T 17 340-359 15 15 15 32 32 32 T T T 18 360-379 25 25 25 53 53 53 T T T 19 380-399 20 20 20 44 44 44 T T T 20 400-419 20 20 20 44 44 44 F F F 21 420-439 26 26 26 54 54 54 T T T 22 440-459 19 19 19 40 40 40 F F F 23 460-479 24 24 24 52 52 52 F F F 24 480-499 25 25 25 53 53 53 F F F 25 500-519 25 25 25 53 53 53 F F F 26 520-539 20 20 20 41 41 41 F F F 27 540-559 26 26 26 54 54 54 T T T 28 560-579 22 22 22 47 47 47 T T T 29 580-599 23 23 23 50 50 50 F F F Max gen: 26, Max flush: 54 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 53 53 53 F F F 1 020-039 27 27 27 55 55 55 F F F 2 040-059 27 27 27 55 55 55 F F F 3 060-079 27 27 27 55 55 55 F F F 4 080-099 27 27 27 55 55 55 F F F 5 100-119 21 21 21 45 45 45 F F F 6 120-139 18 18 18 38 38 38 F F F 7 140-159 25 25 25 53 53 53 F F F 8 160-179 27 27 27 55 55 55 F F F 9 180-199 27 27 27 55 55 55 F F F 10 200-219 27 27 27 55 55 55 F F F 11 220-239 25 25 25 53 53 53 F F F 12 240-259 25 25 25 53 53 53 F F F 13 260-279 25 25 25 53 53 53 F F F 14 280-299 21 21 21 45 45 45 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 27 27 27 55 55 55 F F F 17 340-359 27 27 27 55 55 55 F F F 18 360-379 27 27 27 55 55 55 F F F 19 380-399 27 27 27 55 55 55 F F F 20 400-419 20 20 20 44 44 44 F F F 21 420-439 27 27 27 55 55 55 F F F 22 440-459 19 19 19 40 40 40 F F F 23 460-479 24 24 24 52 52 52 F F F 24 480-499 25 25 25 53 53 53 F F F 25 500-519 25 25 25 53 53 53 F F F 26 520-539 20 20 20 41 41 41 F F F 27 540-559 27 27 27 55 55 55 F F F 28 560-579 27 27 27 55 55 55 F F F 29 580-599 23 23 23 50 50 50 F F F Max gen: 27, Max flush: 55 Begin loop 14 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.90389595Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.947793532Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.948438506Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.948475317Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.948491117Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.948507437Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299} {"msg":"Crucible 1ef1e718-64ea-438f-9eda-611214454c9b has session id: 02539327-2479-4125-a7d2-512313860644","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.948992408Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"Upstairs opts: Upstairs UUID: 1ef1e718-64ea-438f-9eda-611214454c9b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.949040439Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"Crucible stats registered with UUID: 1ef1e718-64ea-438f-9eda-611214454c9b","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.949065699Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.98041463Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.9804463Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299} {"msg":"1ef1e718-64ea-438f-9eda-611214454c9b active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.980516002Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.980538452Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"client":"0","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.980567173Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"client":"0","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.980585113Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"client":"1","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.980602143Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"client":"1","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.980618774Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"client":"2","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.980635444Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"client":"2","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.980651955Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"":"io task","client":"2","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.980678545Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"":"io task","client":"0","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.980711386Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"":"io task","client":"1","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"ds_connection connected from Ok(127.0.0.1:43210)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.980750977Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"":"io task","client":"2","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"ds_connection connected from Ok(127.0.0.1:60677)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.980790488Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"":"io task","client":"1","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"ds_connection connected from Ok(127.0.0.1:60346)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.981021803Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"":"io task","client":"0","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.981635726Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"client":"0","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.981664557Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"client":"0","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.981680867Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"client":"1","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.981695657Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"client":"1","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.981716848Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"client":"2","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.981744128Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"client":"2","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.982259999Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"client":"0","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.98229346Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"client":"1","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.982522685Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"client":"2","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.982834442Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"client":"0","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.982862912Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"client":"0","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.982877983Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.982946614Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"client":"1","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.982967485Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"client":"1","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.982981585Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.983042847Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"client":"2","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.983061927Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"client":"2","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"[0]R flush_numbers[0..12]: [53, 55, 55, 55, 55, 45, 38, 53, 55, 55, 55, 53]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.983076127Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"":"downstairs","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"[0]R generation[0..12]: [25, 27, 27, 27, 27, 21, 18, 25, 27, 27, 27, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.983090008Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"":"downstairs","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.983104348Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"":"downstairs","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"[1]R flush_numbers[0..12]: [53, 55, 55, 55, 55, 45, 38, 53, 55, 55, 55, 53]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.983118158Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"":"downstairs","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"[1]R generation[0..12]: [25, 27, 27, 27, 27, 21, 18, 25, 27, 27, 27, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.983131849Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"":"downstairs","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.983145809Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"":"downstairs","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"[2]R flush_numbers[0..12]: [53, 55, 55, 55, 55, 45, 38, 53, 55, 55, 55, 53]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.983159529Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"":"downstairs","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"[2]R generation[0..12]: [25, 27, 27, 27, 27, 21, 18, 25, 27, 27, 27, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.983173249Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"":"downstairs","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.98318724Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"":"downstairs","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.98320564Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"":"downstairs","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.98322028Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"":"downstairs","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"Next flush: 56","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.983233761Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"":"downstairs","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:51:09.983247001Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"":"downstairs","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.983272421Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.983297632Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"client":"0","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.983323322Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"client":"1","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.983349063Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"client":"2","":"downstairs","session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.983374284Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"session_id":"02539327-2479-4125-a7d2-512313860644"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.983399964Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"1ef1e718-64ea-438f-9eda-611214454c9b is now active with session: 02539327-2479-4125-a7d2-512313860644","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.983424985Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.983441895Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299,"session_id":"02539327-2479-4125-a7d2-512313860644"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:09.983454985Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 246 len 12288 data: 4 4 4 02/30 Read block 191 len 40960 03/30 Write block 95 len 4096 data: 5 04/30 Read block 302 len 32768 05/30 Read block 521 len 28672 06/30 Read block 116 len 16384 07/30 Read block 594 len 4096 08/30 Read block 343 len 8192 09/30 Write block 112 len 4096 data: 3 10/30 Read block 44 len 8192 11/30 Flush 12/30 Write block 21 len 12288 data: 6 5 6 13/30 Read block 446 len 40960 14/30 Read block 443 len 24576 15/30 Write block 422 len 36864 data: 4 4 4 4 4 4 5 4 4 16/30 Write block 148 len 36864 data: 4 4 3 3 3 2 2 2 2 17/30 Flush 18/30 Write block 193 len 12288 data: 5 5 6 19/30 Write block 128 len 4096 data: 3 20/30 Write block 233 len 32768 data: 3 4 4 5 6 4 4 5 21/30 Write block 208 len 20480 data: 4 3 6 6 6 22/30 Write block 333 len 24576 data: 4 4 5 6 5 5 23/30 Read block 437 len 16384 24/30 Read block 487 len 8192 25/30 Write block 538 len 4096 data: 3 26/30 Read block 565 len 8192 27/30 Read block 590 len 36864 28/30 Write block 327 len 40960 data: 5 4 4 5 5 5 5 5 6 7 29/30 Read block 28 len 32768 30/30 Write block 459 len 40960 data: 3 3 3 3 4 4 4 2 3 4 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 3 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:51:10.001763102Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1299} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 53 53 53 F F F 1 020-039 28 28 28 57 57 57 F F F 2 040-059 27 27 27 55 55 55 F F F 3 060-079 27 27 27 55 55 55 F F F 4 080-099 28 28 28 56 56 56 F F F 5 100-119 28 28 28 56 56 56 F F F 6 120-139 18 18 18 38 38 38 T T T 7 140-159 28 28 28 57 57 57 F F F 8 160-179 27 27 27 55 55 55 F F F 9 180-199 27 27 27 55 55 55 T T T 10 200-219 27 27 27 55 55 55 T T T 11 220-239 25 25 25 53 53 53 T T T 12 240-259 28 28 28 56 56 56 T T T 13 260-279 25 25 25 53 53 53 F F F 14 280-299 21 21 21 45 45 45 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 27 27 27 55 55 55 T T T 17 340-359 27 27 27 55 55 55 F F F 18 360-379 27 27 27 55 55 55 F F F 19 380-399 27 27 27 55 55 55 F F F 20 400-419 20 20 20 44 44 44 F F F 21 420-439 28 28 28 57 57 57 F F F 22 440-459 19 19 19 40 40 40 T T T 23 460-479 24 24 24 52 52 52 T T T 24 480-499 25 25 25 53 53 53 F F F 25 500-519 25 25 25 53 53 53 F F F 26 520-539 20 20 20 41 41 41 T T T 27 540-559 27 27 27 55 55 55 F F F 28 560-579 27 27 27 55 55 55 F F F 29 580-599 23 23 23 50 50 50 F F F Max gen: 28, Max flush: 57 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 53 53 53 F F F 1 020-039 28 28 28 57 57 57 F F F 2 040-059 27 27 27 55 55 55 F F F 3 060-079 27 27 27 55 55 55 F F F 4 080-099 28 28 28 56 56 56 F F F 5 100-119 28 28 28 56 56 56 F F F 6 120-139 29 29 29 58 58 58 F F F 7 140-159 28 28 28 57 57 57 F F F 8 160-179 27 27 27 55 55 55 F F F 9 180-199 29 29 29 58 58 58 F F F 10 200-219 29 29 29 58 58 58 F F F 11 220-239 29 29 29 58 58 58 F F F 12 240-259 29 29 29 58 58 58 F F F 13 260-279 25 25 25 53 53 53 F F F 14 280-299 21 21 21 45 45 45 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 29 29 29 58 58 58 F F F 17 340-359 27 27 27 55 55 55 F F F 18 360-379 27 27 27 55 55 55 F F F 19 380-399 27 27 27 55 55 55 F F F 20 400-419 20 20 20 44 44 44 F F F 21 420-439 28 28 28 57 57 57 F F F 22 440-459 29 29 29 58 58 58 F F F 23 460-479 29 29 29 58 58 58 F F F 24 480-499 25 25 25 53 53 53 F F F 25 500-519 25 25 25 53 53 53 F F F 26 520-539 29 29 29 58 58 58 F F F 27 540-559 27 27 27 55 55 55 F F F 28 560-579 27 27 27 55 55 55 F F F 29 580-599 23 23 23 50 50 50 F F F Max gen: 29, Max flush: 58 Begin loop 15 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.288470744Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.333213905Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.33389809Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.33393436Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.333948711Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.333963271Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307} {"msg":"Crucible b2f1639b-be19-4297-94b3-e5b3754c3158 has session id: 9ef1b0b3-4189-46d6-b42a-110808306f27","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.334455582Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"Upstairs opts: Upstairs UUID: b2f1639b-be19-4297-94b3-e5b3754c3158, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.334489092Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"Crucible stats registered with UUID: b2f1639b-be19-4297-94b3-e5b3754c3158","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.334507913Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.365922634Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.365953675Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307} {"msg":"b2f1639b-be19-4297-94b3-e5b3754c3158 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.365977676Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.365993936Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"client":"0","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.366022087Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"client":"0","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.366039727Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"client":"1","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.366058047Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"client":"1","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.366074908Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"client":"2","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.366092008Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"client":"2","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.366108689Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"":"io task","client":"2","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.366126089Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"":"io task","client":"0","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.366143559Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"":"io task","client":"1","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"ds_connection connected from Ok(127.0.0.1:55033)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.366234341Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"":"io task","client":"2","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"ds_connection connected from Ok(127.0.0.1:38653)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.366375714Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"":"io task","client":"1","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"ds_connection connected from Ok(127.0.0.1:61751)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.366415075Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"":"io task","client":"0","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.366898596Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"client":"2","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.366927006Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"client":"2","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.366942147Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"client":"1","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.366956147Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"client":"1","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.366970047Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"client":"0","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.367044889Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"client":"0","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.36755589Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"client":"1","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.367723544Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"client":"0","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.367764914Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"client":"2","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.368125352Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"client":"1","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.368161703Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"client":"1","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.368177553Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.368257655Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"client":"0","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.368278016Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"client":"0","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.368291826Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.368347357Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"client":"2","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.368368458Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"client":"2","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"[0]R flush_numbers[0..12]: [53, 57, 55, 55, 56, 56, 58, 57, 55, 58, 58, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.368382858Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"":"downstairs","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"[0]R generation[0..12]: [25, 28, 27, 27, 28, 28, 29, 28, 27, 29, 29, 29]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.368396828Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"":"downstairs","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.368411179Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"":"downstairs","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"[1]R flush_numbers[0..12]: [53, 57, 55, 55, 56, 56, 58, 57, 55, 58, 58, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.368425109Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"":"downstairs","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"[1]R generation[0..12]: [25, 28, 27, 27, 28, 28, 29, 28, 27, 29, 29, 29]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.368439029Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"":"downstairs","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.368453189Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"":"downstairs","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"[2]R flush_numbers[0..12]: [53, 57, 55, 55, 56, 56, 58, 57, 55, 58, 58, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.36846693Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"":"downstairs","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"[2]R generation[0..12]: [25, 28, 27, 27, 28, 28, 29, 28, 27, 29, 29, 29]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.36848087Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"":"downstairs","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.36849482Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"":"downstairs","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.368507961Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"":"downstairs","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.368521371Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"":"downstairs","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"Next flush: 59","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.368534501Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"":"downstairs","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.368547821Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.368571752Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.368583992Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"client":"0","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.368597343Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"client":"1","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.368610483Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"client":"2","":"downstairs","session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.368623853Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.368636313Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"b2f1639b-be19-4297-94b3-e5b3754c3158 is now active with session: 9ef1b0b3-4189-46d6-b42a-110808306f27","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.368649374Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.368661494Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307,"session_id":"9ef1b0b3-4189-46d6-b42a-110808306f27"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:14.368673804Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 519 len 12288 data: 5 4 4 02/30 Write block 257 len 24576 data: 3 3 3 2 2 2 03/30 Read block 66 len 4096 04/30 Write block 526 len 4096 data: 5 05/30 Write block 277 len 8192 data: 4 3 06/30 Read block 154 len 32768 07/30 Read block 492 len 36864 08/30 Write block 445 len 12288 data: 3 3 3 09/30 Read block 275 len 20480 10/30 Read block 517 len 12288 11/30 Read block 76 len 36864 12/30 Write block 121 len 16384 data: 5 5 6 4 13/30 Read block 171 len 28672 14/30 Write block 115 len 12288 data: 4 4 4 15/30 Read block 446 len 36864 16/30 Read block 355 len 32768 17/30 Write block 381 len 16384 data: 4 3 2 2 18/30 Read block 527 len 4096 19/30 Read block 412 len 20480 20/30 Read block 443 len 40960 21/30 Write block 495 len 36864 data: 4 3 3 3 3 4 3 2 3 22/30 Write block 342 len 28672 data: 2 3 3 3 3 3 3 23/30 Write block 500 len 32768 data: 5 4 3 4 3 4 5 6 24/30 Read block 91 len 40960 25/30 Write block 310 len 24576 data: 2 2 2 2 2 2 26/30 Read block 162 len 36864 27/30 Write block 2 len 20480 data: 2 2 2 3 3 28/30 Write block 240 len 36864 data: 6 5 3 4 3 4 5 5 5 29/30 Read block 48 len 28672 30/30 Read block 282 len 12288 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Read 1 Done Done Done false 1003 Acked Write 1 Done Done Done false 1004 Acked Write 2 Done Done Done false 1005 Acked Read 8 Done Done Done false 1006 Acked Read 9 Done Done Done false 1007 Acked Write 3 Done Done Done false 1008 Acked Read 5 Done Done Done false 1009 Acked Read 3 Done Done Done false 1010 Acked Read 9 Done Done Done false 1011 Acked Write 4 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Write 3 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:51:14.387700187Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1307} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 53 53 53 T T T 1 020-039 28 28 28 57 57 57 F F F 2 040-059 27 27 27 55 55 55 F F F 3 060-079 27 27 27 55 55 55 F F F 4 080-099 28 28 28 56 56 56 F F F 5 100-119 28 28 28 56 56 56 T T T 6 120-139 29 29 29 58 58 58 T T T 7 140-159 28 28 28 57 57 57 F F F 8 160-179 27 27 27 55 55 55 F F F 9 180-199 29 29 29 58 58 58 F F F 10 200-219 29 29 29 58 58 58 F F F 11 220-239 29 29 29 58 58 58 F F F 12 240-259 29 29 29 58 58 58 T T T 13 260-279 25 25 25 53 53 53 T T T 14 280-299 21 21 21 45 45 45 F F F 15 300-319 2 2 2 3 3 3 T T T 16 320-339 29 29 29 58 58 58 F F F 17 340-359 27 27 27 55 55 55 T T T 18 360-379 27 27 27 55 55 55 F F F 19 380-399 27 27 27 55 55 55 T T T 20 400-419 20 20 20 44 44 44 F F F 21 420-439 28 28 28 57 57 57 F F F 22 440-459 29 29 29 58 58 58 T T T 23 460-479 29 29 29 58 58 58 F F F 24 480-499 25 25 25 53 53 53 T T T 25 500-519 25 25 25 53 53 53 T T T 26 520-539 29 29 29 58 58 58 T T T 27 540-559 27 27 27 55 55 55 F F F 28 560-579 27 27 27 55 55 55 F F F 29 580-599 23 23 23 50 50 50 F F F Max gen: 29, Max flush: 58 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 59 59 59 F F F 1 020-039 28 28 28 57 57 57 F F F 2 040-059 27 27 27 55 55 55 F F F 3 060-079 27 27 27 55 55 55 F F F 4 080-099 28 28 28 56 56 56 F F F 5 100-119 30 30 30 59 59 59 F F F 6 120-139 30 30 30 59 59 59 F F F 7 140-159 28 28 28 57 57 57 F F F 8 160-179 27 27 27 55 55 55 F F F 9 180-199 29 29 29 58 58 58 F F F 10 200-219 29 29 29 58 58 58 F F F 11 220-239 29 29 29 58 58 58 F F F 12 240-259 30 30 30 59 59 59 F F F 13 260-279 30 30 30 59 59 59 F F F 14 280-299 21 21 21 45 45 45 F F F 15 300-319 30 30 30 59 59 59 F F F 16 320-339 29 29 29 58 58 58 F F F 17 340-359 30 30 30 59 59 59 F F F 18 360-379 27 27 27 55 55 55 F F F 19 380-399 30 30 30 59 59 59 F F F 20 400-419 20 20 20 44 44 44 F F F 21 420-439 28 28 28 57 57 57 F F F 22 440-459 30 30 30 59 59 59 F F F 23 460-479 29 29 29 58 58 58 F F F 24 480-499 30 30 30 59 59 59 F F F 25 500-519 30 30 30 59 59 59 F F F 26 520-539 30 30 30 59 59 59 F F F 27 540-559 27 27 27 55 55 55 F F F 28 560-579 27 27 27 55 55 55 F F F 29 580-599 23 23 23 50 50 50 F F F Max gen: 30, Max flush: 59 Begin loop 16 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.686418559Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:51:18.728969792Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.72934613Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.761290843Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.761891556Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.761926317Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.761941077Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.761955818Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315} {"msg":"Crucible 89724264-372c-4451-90ee-3f1279142e87 has session id: e5dc378d-9fce-4946-8b2c-79b55fd8e19b","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.762449188Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"Upstairs opts: Upstairs UUID: 89724264-372c-4451-90ee-3f1279142e87, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.762496649Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"Crucible stats registered with UUID: 89724264-372c-4451-90ee-3f1279142e87","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.76252121Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.793362759Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.79339039Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315} {"msg":"89724264-372c-4451-90ee-3f1279142e87 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.793447931Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.793482861Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"client":"0","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.793517892Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"client":"0","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.793561173Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"client":"1","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.793593324Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"client":"1","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.793623764Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"client":"2","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.793656055Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"client":"2","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.793687826Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"":"io task","client":"0","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.793721947Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"":"io task","client":"1","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.793758627Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"":"io task","client":"2","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"ds_connection connected from Ok(127.0.0.1:47969)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.793794918Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"":"io task","client":"1","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"ds_connection connected from Ok(127.0.0.1:63435)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.79389555Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"":"io task","client":"0","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"ds_connection connected from Ok(127.0.0.1:56156)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.793931051Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"":"io task","client":"2","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.794490803Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"client":"1","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.794520444Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"client":"1","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.794668547Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"client":"2","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.794710458Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"client":"2","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.794742199Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"client":"0","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.79481042Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"client":"0","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.795180668Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"client":"1","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.795222489Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"client":"2","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.795481095Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"client":"0","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.795764001Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"client":"1","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.795804272Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"client":"1","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.795836072Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.795864753Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"client":"2","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.795886414Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"client":"2","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.795901324Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.795998546Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"client":"0","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.796018667Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"client":"0","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"[0]R flush_numbers[0..12]: [59, 57, 55, 55, 56, 59, 59, 57, 55, 58, 58, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.796032987Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"":"downstairs","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"[0]R generation[0..12]: [30, 28, 27, 27, 28, 30, 30, 28, 27, 29, 29, 29]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.796047007Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"":"downstairs","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.796061037Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"":"downstairs","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"[1]R flush_numbers[0..12]: [59, 57, 55, 55, 56, 59, 59, 57, 55, 58, 58, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.796074798Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"":"downstairs","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"[1]R generation[0..12]: [30, 28, 27, 27, 28, 30, 30, 28, 27, 29, 29, 29]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.796088378Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"":"downstairs","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.796102278Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"":"downstairs","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"[2]R flush_numbers[0..12]: [59, 57, 55, 55, 56, 59, 59, 57, 55, 58, 58, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.796115799Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"":"downstairs","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"[2]R generation[0..12]: [30, 28, 27, 27, 28, 30, 30, 28, 27, 29, 29, 29]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.796129329Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"":"downstairs","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.796143459Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"":"downstairs","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"Max found gen is 31","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.796156559Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"":"downstairs","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"Generation requested: 32 >= found:31","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.79617263Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"":"downstairs","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"Next flush: 60","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.79618617Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"":"downstairs","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.796199431Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"":"downstairs","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.796221761Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.796234111Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"client":"0","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.796247692Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"client":"1","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.796261152Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"client":"2","":"downstairs","session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.796274622Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.796287062Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"89724264-372c-4451-90ee-3f1279142e87 is now active with session: e5dc378d-9fce-4946-8b2c-79b55fd8e19b","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.796299893Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:18.796311913Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315,"session_id":"e5dc378d-9fce-4946-8b2c-79b55fd8e19b"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2024-11-28T04:51:18.796324193Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 524 len 12288 data: 3 3 6 02/30 Write block 52 len 28672 data: 5 5 6 6 8 8 8 03/30 Write block 114 len 12288 data: 4 5 5 04/30 Write block 183 len 36864 data: 5 5 4 6 6 6 6 5 5 05/30 Write block 450 len 20480 data: 2 4 4 4 4 06/30 Read block 576 len 40960 07/30 Write block 342 len 4096 data: 3 08/30 Flush 09/30 Write block 339 len 32768 data: 4 4 2 4 4 4 4 4 10/30 Read block 586 len 8192 11/30 Write block 284 len 28672 data: 4 4 4 3 2 2 2 12/30 Write block 480 len 36864 data: 2 2 2 2 2 2 3 3 3 13/30 Read block 509 len 16384 14/30 Write block 278 len 40960 data: 4 2 2 2 4 3 5 5 5 4 15/30 Flush 16/30 Write block 199 len 8192 data: 5 6 17/30 Read block 26 len 8192 18/30 Write block 454 len 12288 data: 5 3 3 19/30 Write block 15 len 12288 data: 5 5 5 20/30 Read block 81 len 28672 21/30 Read block 77 len 4096 22/30 Flush 23/30 Flush 24/30 Write block 49 len 8192 data: 5 5 25/30 Read block 588 len 8192 26/30 Read block 242 len 40960 27/30 Write block 18 len 8192 data: 4 7 28/30 Write block 421 len 20480 data: 3 5 5 5 5 29/30 Write block 395 len 28672 data: 4 4 3 4 5 4 4 30/30 Read block 595 len 20480 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 2 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 5 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:51:18.814115159Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1315} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 62 62 62 T T T 1 020-039 28 28 28 57 57 57 F F F 2 040-059 32 32 32 60 60 60 T T T 3 060-079 27 27 27 55 55 55 F F F 4 080-099 28 28 28 56 56 56 F F F 5 100-119 32 32 32 60 60 60 F F F 6 120-139 30 30 30 59 59 59 F F F 7 140-159 28 28 28 57 57 57 F F F 8 160-179 27 27 27 55 55 55 F F F 9 180-199 32 32 32 62 62 62 F F F 10 200-219 32 32 32 62 62 62 F F F 11 220-239 29 29 29 58 58 58 F F F 12 240-259 30 30 30 59 59 59 F F F 13 260-279 32 32 32 61 61 61 F F F 14 280-299 32 32 32 61 61 61 F F F 15 300-319 30 30 30 59 59 59 F F F 16 320-339 32 32 32 61 61 61 F F F 17 340-359 32 32 32 61 61 61 F F F 18 360-379 27 27 27 55 55 55 F F F 19 380-399 30 30 30 59 59 59 T T T 20 400-419 20 20 20 44 44 44 T T T 21 420-439 28 28 28 57 57 57 T T T 22 440-459 32 32 32 62 62 62 F F F 23 460-479 29 29 29 58 58 58 F F F 24 480-499 32 32 32 61 61 61 F F F 25 500-519 30 30 30 59 59 59 F F F 26 520-539 32 32 32 60 60 60 F F F 27 540-559 27 27 27 55 55 55 F F F 28 560-579 27 27 27 55 55 55 F F F 29 580-599 23 23 23 50 50 50 F F F Max gen: 32, Max flush: 62 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 63 63 63 F F F 1 020-039 28 28 28 57 57 57 F F F 2 040-059 33 33 33 63 63 63 F F F 3 060-079 27 27 27 55 55 55 F F F 4 080-099 28 28 28 56 56 56 F F F 5 100-119 32 32 32 60 60 60 F F F 6 120-139 30 30 30 59 59 59 F F F 7 140-159 28 28 28 57 57 57 F F F 8 160-179 27 27 27 55 55 55 F F F 9 180-199 32 32 32 62 62 62 F F F 10 200-219 32 32 32 62 62 62 F F F 11 220-239 29 29 29 58 58 58 F F F 12 240-259 30 30 30 59 59 59 F F F 13 260-279 32 32 32 61 61 61 F F F 14 280-299 32 32 32 61 61 61 F F F 15 300-319 30 30 30 59 59 59 F F F 16 320-339 32 32 32 61 61 61 F F F 17 340-359 32 32 32 61 61 61 F F F 18 360-379 27 27 27 55 55 55 F F F 19 380-399 33 33 33 63 63 63 F F F 20 400-419 33 33 33 63 63 63 F F F 21 420-439 33 33 33 63 63 63 F F F 22 440-459 32 32 32 62 62 62 F F F 23 460-479 29 29 29 58 58 58 F F F 24 480-499 32 32 32 61 61 61 F F F 25 500-519 30 30 30 59 59 59 F F F 26 520-539 32 32 32 60 60 60 F F F 27 540-559 27 27 27 55 55 55 F F F 28 560-579 27 27 27 55 55 55 F F F 29 580-599 23 23 23 50 50 50 F F F Max gen: 33, Max flush: 63 Begin loop 17 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.10816579Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:51:23.151594642Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.15195812Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.184797882Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.185373455Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.185406126Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.185420306Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.185434506Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323} {"msg":"Crucible 7b23237e-d76d-42d5-944f-83e444bbcc63 has session id: c23fd587-7e43-4803-8c42-e81d5abdb3e1","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.185824485Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"Upstairs opts: Upstairs UUID: 7b23237e-d76d-42d5-944f-83e444bbcc63, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.185872186Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"Crucible stats registered with UUID: 7b23237e-d76d-42d5-944f-83e444bbcc63","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.185902026Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.217494332Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.217528383Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323} {"msg":"7b23237e-d76d-42d5-944f-83e444bbcc63 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.217593754Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.217615474Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"client":"0","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.217646255Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"client":"0","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.217665535Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"client":"1","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.217715396Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"client":"1","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.217749087Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"client":"2","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.217780498Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"client":"2","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.217812219Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"":"io task","client":"2","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.218024973Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"":"io task","client":"0","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.218063694Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"":"io task","client":"1","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:43832)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.218097275Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"":"io task","client":"0","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:44743)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.218129125Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"":"io task","client":"2","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:48049)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.218156156Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"":"io task","client":"1","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.218612776Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"client":"1","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.218706548Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"client":"1","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.21877241Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"client":"2","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.21880548Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"client":"2","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.218850901Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"client":"0","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.218899252Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"client":"0","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.219364302Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"client":"2","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.219394973Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"client":"1","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.219422644Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"client":"0","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.219883603Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"client":"2","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.219981446Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"client":"2","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.220017006Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.220041767Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"client":"1","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.220064967Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"client":"1","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.220089028Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.220111039Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"client":"0","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.220140689Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"client":"0","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"[0]R flush_numbers[0..12]: [63, 57, 63, 55, 56, 60, 59, 57, 55, 62, 62, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.22016599Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"":"downstairs","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"[0]R generation[0..12]: [33, 28, 33, 27, 28, 32, 30, 28, 27, 32, 32, 29]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.22018998Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"":"downstairs","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.220216691Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"":"downstairs","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"[1]R flush_numbers[0..12]: [63, 57, 63, 55, 56, 60, 59, 57, 55, 62, 62, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.220244361Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"":"downstairs","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"[1]R generation[0..12]: [33, 28, 33, 27, 28, 32, 30, 28, 27, 32, 32, 29]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.220269882Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"":"downstairs","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.220297252Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"Wait for a query_work_queue command to finish before sending IO ":"downstairs","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"[2]R flush_numbers[0..12]: [63, 57, 63, 55, 56, 60, 59, 57, 55, 62, 62, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.220336123Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"":"downstairs","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"[2]R generation[0..12]: [33, 28, 33, 27, 28, 32, 30, 28, 27, 32, 32, 29]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.220366954Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"":"downstairs","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.220397615Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"":"downstairs","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.220427795Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"":"downstairs","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.220457716Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"":"downstairs","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"Next flush: 64","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-28T04:51:23.220486957Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"":"downstairs","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.220520047Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"":"downstairs","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.220537538Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.220550428Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"client":"0","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.220563928Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"client":"1","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.220577049Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"client":"2","":"downstairs","session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.220590289Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.220602579Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"7b23237e-d76d-42d5-944f-83e444bbcc63 is now active with session: c23fd587-7e43-4803-8c42-e81d5abdb3e1","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.22061551Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.22062857Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323,"session_id":"c23fd587-7e43-4803-8c42-e81d5abdb3e1"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:23.22064089Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 42 len 40960 data: 3 3 4 5 4 4 4 6 6 5 02/30 Read block 174 len 24576 03/30 Read block 30 len 8192 04/30 Read block 246 len 36864 05/30 Read block 102 len 20480 06/30 Write block 254 len 20480 data: 3 3 3 4 4 07/30 Write block 456 len 8192 data: 4 3 08/30 Write block 37 len 12288 data: 2 2 2 09/30 Write block 537 len 12288 data: 3 4 3 10/30 Write block 98 len 24576 data: 4 3 3 3 4 4 11/30 Read block 44 len 28672 12/30 Write block 285 len 32768 data: 6 6 5 3 3 3 2 3 13/30 Read block 186 len 28672 14/30 Read block 38 len 4096 15/30 Write block 480 len 12288 data: 3 3 3 16/30 Read block 129 len 32768 17/30 Flush 18/30 Write block 303 len 24576 data: 2 2 2 2 2 2 19/30 Write block 553 len 32768 data: 5 5 5 5 4 3 3 2 20/30 Read block 420 len 36864 21/30 Write block 428 len 8192 data: 6 5 22/30 Flush 23/30 Write block 385 len 20480 data: 2 2 3 3 3 24/30 Write block 60 len 20480 data: 8 7 4 3 4 25/30 Flush 26/30 Write block 71 len 8192 data: 8 7 27/30 Write block 387 len 12288 data: 4 4 4 28/30 Write block 583 len 40960 data: 2 2 2 2 2 2 2 3 3 3 29/30 Read block 252 len 24576 30/30 Read block 402 len 16384 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 2 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 5 4 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:51:23.239299475Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1323} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 63 63 63 F F F 1 020-039 34 34 34 64 64 64 F F F 2 040-059 34 34 34 64 64 64 F F F 3 060-079 34 34 34 66 66 66 T T T 4 080-099 34 34 34 64 64 64 F F F 5 100-119 34 34 34 64 64 64 F F F 6 120-139 30 30 30 59 59 59 F F F 7 140-159 28 28 28 57 57 57 F F F 8 160-179 27 27 27 55 55 55 F F F 9 180-199 32 32 32 62 62 62 F F F 10 200-219 32 32 32 62 62 62 F F F 11 220-239 29 29 29 58 58 58 F F F 12 240-259 34 34 34 64 64 64 F F F 13 260-279 32 32 32 61 61 61 F F F 14 280-299 34 34 34 64 64 64 F F F 15 300-319 34 34 34 65 65 65 F F F 16 320-339 32 32 32 61 61 61 F F F 17 340-359 32 32 32 61 61 61 F F F 18 360-379 27 27 27 55 55 55 F F F 19 380-399 34 34 34 66 66 66 T T T 20 400-419 33 33 33 63 63 63 F F F 21 420-439 34 34 34 65 65 65 F F F 22 440-459 34 34 34 64 64 64 F F F 23 460-479 29 29 29 58 58 58 F F F 24 480-499 34 34 34 64 64 64 F F F 25 500-519 30 30 30 59 59 59 F F F 26 520-539 34 34 34 64 64 64 F F F 27 540-559 34 34 34 65 65 65 F F F 28 560-579 34 34 34 65 65 65 F F F 29 580-599 23 23 23 50 50 50 T T T Max gen: 34, Max flush: 66 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 63 63 63 F F F 1 020-039 34 34 34 64 64 64 F F F 2 040-059 34 34 34 64 64 64 F F F 3 060-079 35 35 35 67 67 67 F F F 4 080-099 34 34 34 64 64 64 F F F 5 100-119 34 34 34 64 64 64 F F F 6 120-139 30 30 30 59 59 59 F F F 7 140-159 28 28 28 57 57 57 F F F 8 160-179 27 27 27 55 55 55 F F F 9 180-199 32 32 32 62 62 62 F F F 10 200-219 32 32 32 62 62 62 F F F 11 220-239 29 29 29 58 58 58 F F F 12 240-259 34 34 34 64 64 64 F F F 13 260-279 32 32 32 61 61 61 F F F 14 280-299 34 34 34 64 64 64 F F F 15 300-319 34 34 34 65 65 65 F F F 16 320-339 32 32 32 61 61 61 F F F 17 340-359 32 32 32 61 61 61 F F F 18 360-379 27 27 27 55 55 55 F F F 19 380-399 35 35 35 67 67 67 F F F 20 400-419 33 33 33 63 63 63 F F F 21 420-439 34 34 34 65 65 65 F F F 22 440-459 34 34 34 64 64 64 F F F 23 460-479 29 29 29 58 58 58 F F F 24 480-499 34 34 34 64 64 64 F F F 25 500-519 30 30 30 59 59 59 F F F 26 520-539 34 34 34 64 64 64 F F F 27 540-559 34 34 34 65 65 65 F F F 28 560-579 34 34 34 65 65 65 F F F 29 580-599 35 35 35 67 67 67 F F F Max gen: 35, Max flush: 67 Begin loop 18 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.525483125Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:51:27.568002017Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.568362375Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.59993606Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.600499082Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.600534833Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.600549303Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.600563804Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331} {"msg":"Crucible 4926970b-b78d-4a9d-990b-c88e5012f4d9 has session id: 44ee6a7f-ae59-4fa5-b3aa-c49b33e636be","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.600970153Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"Upstairs opts: Upstairs UUID: 4926970b-b78d-4a9d-990b-c88e5012f4d9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.601015434Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"Crucible stats registered with UUID: 4926970b-b78d-4a9d-990b-c88e5012f4d9","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.601034674Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.63176034Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.631795481Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331} {"msg":"4926970b-b78d-4a9d-990b-c88e5012f4d9 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.631822012Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.631845202Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"client":"0","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.631876303Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"client":"0","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.631897013Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"client":"1","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.631916304Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"client":"1","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.631939104Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"client":"2","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.631969835Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"client":"2","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.632001686Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"":"io task","client":"2","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.632033346Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"":"io task","client":"0","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.632288642Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"":"io task","client":"1","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"ds_connection connected from Ok(127.0.0.1:40530)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.632324843Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"":"io task","client":"2","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"ds_connection connected from Ok(127.0.0.1:58152)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.632347063Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"":"io task","client":"1","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"ds_connection connected from Ok(127.0.0.1:33718)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.632379374Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"":"io task","client":"0","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.632744332Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"client":"2","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.632785273Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"client":"2","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.632815353Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"client":"1","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.632848174Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"client":"1","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.633080809Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"client":"0","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.63310623Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"client":"0","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.633548589Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"client":"1","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.63359179Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"client":"2","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.633624961Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"client":"0","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.634083161Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"client":"2","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.634110921Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"client":"2","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.634125482Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.634202333Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"client":"1","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.634225534Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"client":"1","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.634239384Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.634252424Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"client":"0","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.634266115Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"client":"0","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"[0]R flush_numbers[0..12]: [63, 64, 64, 67, 64, 64, 59, 57, 55, 62, 62, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.634279855Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"":"downstairs","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"[0]R generation[0..12]: [33, 34, 34, 35, 34, 34, 30, 28, 27, 32, 32, 29]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.634296745Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"":"downstairs","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.634312456Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"":"downstairs","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"[1]R flush_numbers[0..12]: [63, 64, 64, 67, 64, 64, 59, 57, 55, 62, 62, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.634326256Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"":"downstairs","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"[1]R generation[0..12]: [33, 34, 34, 35, 34, 34, 30, 28, 27, 32, 32, 29]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.634339856Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"":"downstairs","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.634353657Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"":"downstairs","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"[2]R flush_numbers[0..12]: [63, 64, 64, 67, 64, 64, 59, 57, 55, 62, 62, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.634367267Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"":"downstairs","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"[2]R generation[0..12]: [33, 34, 34, 35, 34, 34, 30, 28, 27, 32, 32, 29]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.634380777Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"":"downstairs","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.634403948Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"":"downstairs","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.634436999Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"":"downstairs","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.634462229Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"":"downstairs","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"Next flush: 68","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.63448374Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"":"downstairs","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.63450854Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"":"downstairs","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.63453219Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-28T04:51:27.634554701Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"client":"0","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.634580782Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"client":"1","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.634606602Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"client":"2","":"downstairs","session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.634624892Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.634638663Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"4926970b-b78d-4a9d-990b-c88e5012f4d9 is now active with session: 44ee6a7f-ae59-4fa5-b3aa-c49b33e636be","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.634652013Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.634664163Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331,"session_id":"44ee6a7f-ae59-4fa5-b3aa-c49b33e636be"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:27.634676474Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 168 len 16384 data: 3 3 2 3 02/30 Read block 166 len 28672 03/30 Write block 327 len 36864 data: 6 5 5 6 6 6 6 6 7 04/30 Read block 435 len 20480 05/30 Read block 136 len 32768 06/30 Write block 372 len 24576 data: 5 3 3 4 4 5 07/30 Read block 125 len 24576 08/30 Write block 586 len 4096 data: 3 09/30 Read block 493 len 8192 10/30 Flush 11/30 Write block 319 len 32768 data: 2 2 2 2 2 4 5 5 12/30 Read block 392 len 36864 13/30 Read block 579 len 32768 14/30 Write block 185 len 40960 data: 5 7 7 7 7 6 6 5 6 6 15/30 Flush 16/30 Write block 474 len 24576 data: 4 4 3 2 2 2 17/30 Write block 546 len 8192 data: 4 4 18/30 Flush 19/30 Write block 1 len 36864 data: 2 3 3 3 4 4 3 3 2 20/30 Read block 157 len 16384 21/30 Write block 531 len 40960 data: 2 2 3 4 3 3 4 5 4 3 22/30 Write block 111 len 20480 data: 3 4 4 5 6 23/30 Read block 77 len 28672 24/30 Write block 293 len 16384 data: 3 3 2 2 25/30 Write block 170 len 28672 data: 3 4 3 3 4 4 4 26/30 Read block 572 len 24576 27/30 Read block 451 len 28672 28/30 Read block 504 len 24576 29/30 Write block 152 len 4096 data: 4 30/30 Write block 93 len 12288 data: 4 4 6 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 9 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:51:27.654743429Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1331} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 63 63 63 T T T 1 020-039 34 34 34 64 64 64 F F F 2 040-059 34 34 34 64 64 64 F F F 3 060-079 35 35 35 67 67 67 F F F 4 080-099 34 34 34 64 64 64 T T T 5 100-119 34 34 34 64 64 64 T T T 6 120-139 30 30 30 59 59 59 F F F 7 140-159 28 28 28 57 57 57 T T T 8 160-179 36 36 36 68 68 68 T T T 9 180-199 36 36 36 69 69 69 F F F 10 200-219 32 32 32 62 62 62 F F F 11 220-239 29 29 29 58 58 58 F F F 12 240-259 34 34 34 64 64 64 F F F 13 260-279 32 32 32 61 61 61 F F F 14 280-299 34 34 34 64 64 64 T T T 15 300-319 36 36 36 69 69 69 F F F 16 320-339 36 36 36 69 69 69 F F F 17 340-359 32 32 32 61 61 61 F F F 18 360-379 36 36 36 68 68 68 F F F 19 380-399 35 35 35 67 67 67 F F F 20 400-419 33 33 33 63 63 63 F F F 21 420-439 34 34 34 65 65 65 F F F 22 440-459 34 34 34 64 64 64 F F F 23 460-479 36 36 36 70 70 70 F F F 24 480-499 34 34 34 64 64 64 F F F 25 500-519 30 30 30 59 59 59 F F F 26 520-539 34 34 34 64 64 64 T T T 27 540-559 36 36 36 70 70 70 T T T 28 560-579 34 34 34 65 65 65 F F F 29 580-599 36 36 36 68 68 68 F F F Max gen: 36, Max flush: 70 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 71 71 71 F F F 1 020-039 34 34 34 64 64 64 F F F 2 040-059 34 34 34 64 64 64 F F F 3 060-079 35 35 35 67 67 67 F F F 4 080-099 37 37 37 71 71 71 F F F 5 100-119 37 37 37 71 71 71 F F F 6 120-139 30 30 30 59 59 59 F F F 7 140-159 37 37 37 71 71 71 F F F 8 160-179 37 37 37 71 71 71 F F F 9 180-199 36 36 36 69 69 69 F F F 10 200-219 32 32 32 62 62 62 F F F 11 220-239 29 29 29 58 58 58 F F F 12 240-259 34 34 34 64 64 64 F F F 13 260-279 32 32 32 61 61 61 F F F 14 280-299 37 37 37 71 71 71 F F F 15 300-319 36 36 36 69 69 69 F F F 16 320-339 36 36 36 69 69 69 F F F 17 340-359 32 32 32 61 61 61 F F F 18 360-379 36 36 36 68 68 68 F F F 19 380-399 35 35 35 67 67 67 F F F 20 400-419 33 33 33 63 63 63 F F F 21 420-439 34 34 34 65 65 65 F F F 22 440-459 34 34 34 64 64 64 F F F 23 460-479 36 36 36 70 70 70 F F F 24 480-499 34 34 34 64 64 64 F F F 25 500-519 30 30 30 59 59 59 F F F 26 520-539 37 37 37 71 71 71 F F F 27 540-559 37 37 37 71 71 71 F F F 28 560-579 34 34 34 65 65 65 F F F 29 580-599 36 36 36 68 68 68 F F F Max gen: 37, Max flush: 71 Begin loop 19 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:31.968553078Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.012583093Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.013295379Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.013331419Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.01334617Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.01336146Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339} {"msg":"Crucible 839fad0a-539e-4a82-8e5c-d95f4355bf53 has session id: 045db552-0bb2-42ab-9728-0c032b444644","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.013995274Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"Upstairs opts: Upstairs UUID: 839fad0a-539e-4a82-8e5c-d95f4355bf53, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.014030185Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"Crucible stats registered with UUID: 839fad0a-539e-4a82-8e5c-d95f4355bf53","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.014046045Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.044794202Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.044825723Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339} {"msg":"839fad0a-539e-4a82-8e5c-d95f4355bf53 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.044874614Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.044894404Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"client":"0","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.044921805Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"client":"0","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.044939395Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"client":"1","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.044961296Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"client":"1","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.044980146Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"client":"2","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.045011737Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"client":"2","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.045045087Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"":"io task","client":"2","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.045080968Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"":"io task","client":"0","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.045116859Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"":"io task","client":"1","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"ds_connection connected from Ok(127.0.0.1:40637)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.045321693Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"":"io task","client":"0","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"ds_connection connected from Ok(127.0.0.1:64118)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.045363305Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"":"io task","client":"1","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"ds_connection connected from Ok(127.0.0.1:62769)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.045391545Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"":"io task","client":"2","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.045779423Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"client":"1","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.045826735Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"client":"1","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.045858765Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"client":"0","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.045892626Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"client":"0","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.046003278Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"client":"2","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.046025009Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"client":"2","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.046516259Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"client":"1","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.04654594Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"client":"0","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.04656292Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"client":"2","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.047085732Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"client":"1","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.047112522Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"client":"1","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.047127083Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.047206525Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"client":"0","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.047225695Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"client":"0","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.047239515Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.047252145Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"client":"2","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.047265716Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"client":"2","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"[0]R flush_numbers[0..12]: [71, 64, 64, 67, 71, 71, 59, 71, 71, 69, 62, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.047279706Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"":"downstairs","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"[0]R generation[0..12]: [37, 34, 34, 35, 37, 37, 30, 37, 37, 36, 32, 29]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.047293916Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"":"downstairs","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.047308117Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"":"downstairs","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"[1]R flush_numbers[0..12]: [71, 64, 64, 67, 71, 71, 59, 71, 71, 69, 62, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.047321947Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"":"downstairs","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"[1]R generation[0..12]: [37, 34, 34, 35, 37, 37, 30, 37, 37, 36, 32, 29]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.047339487Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"":"downstairs","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.047354118Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"":"downstairs","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"[2]R flush_numbers[0..12]: [71, 64, 64, 67, 71, 71, 59, 71, 71, 69, 62, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.047367788Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"":"downstairs","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"[2]R generation[0..12]: [37, 34, 34, 35, 37, 37, 30, 37, 37, 36, 32, 29]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.047381558Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"":"downstairs","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.047395618Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"":"downstairs","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.047408759Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"":"downstairs","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.047422299Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"":"downstairs","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"Next flush: 72","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2024-11-28T04:51:32.04747987Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"":"downstairs","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.047511881Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"":"downstairs","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.047528561Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.047541362Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"client":"0","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.047555182Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"client":"1","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.047568492Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"client":"2","":"downstairs","session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.047582082Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.047594653Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"839fad0a-539e-4a82-8e5c-d95f4355bf53 is now active with session: 045db552-0bb2-42ab-9728-0c032b444644","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.047611053Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.047623734Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339,"session_id":"045db552-0bb2-42ab-9728-0c032b444644"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:32.047636394Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 285 len 32768 data: 7 7 6 4 4 4 3 4 02/30 Write block 292 len 4096 data: 5 03/30 Read block 179 len 4096 04/30 Read block 401 len 4096 05/30 Read block 176 len 36864 06/30 Read block 128 len 36864 07/30 Write block 469 len 40960 data: 4 4 3 3 3 5 5 4 3 3 08/30 Write block 301 len 36864 data: 3 3 3 3 3 3 3 3 2 09/30 Flush 10/30 Write block 344 len 36864 data: 5 5 5 4 4 3 4 5 6 11/30 Read block 440 len 28672 12/30 Read block 403 len 16384 13/30 Write block 474 len 12288 data: 6 6 5 14/30 Write block 565 len 40960 data: 2 2 3 3 5 5 4 4 5 5 15/30 Read block 269 len 40960 16/30 Read block 347 len 4096 17/30 Read block 201 len 20480 18/30 Write block 250 len 20480 data: 4 4 4 4 4 19/30 Write block 378 len 8192 data: 5 5 20/30 Read block 72 len 24576 21/30 Write block 338 len 32768 data: 6 5 5 3 5 5 6 6 22/30 Write block 99 len 20480 data: 4 4 4 5 5 23/30 Read block 560 len 36864 24/30 Read block 395 len 40960 25/30 Read block 133 len 16384 26/30 Write block 477 len 28672 data: 4 4 3 4 4 4 3 27/30 Write block 218 len 16384 data: 2 3 3 4 28/30 Read block 468 len 40960 29/30 Write block 583 len 32768 data: 3 3 3 4 3 3 3 4 30/30 Write block 575 len 28672 data: 6 5 5 4 4 3 2 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 9 Done Done Done false 1010 Acked Read 7 Done Done Done false 1011 Acked Read 4 Done Done Done false 1012 Acked Write 3 Done Done Done false 1013 Acked Write 10 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 19 19 19 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1008 1008 1008 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:51:32.067380022Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1339} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 71 71 71 F F F 1 020-039 34 34 34 64 64 64 F F F 2 040-059 34 34 34 64 64 64 F F F 3 060-079 35 35 35 67 67 67 F F F 4 080-099 37 37 37 71 71 71 T T T 5 100-119 37 37 37 71 71 71 T T T 6 120-139 30 30 30 59 59 59 F F F 7 140-159 37 37 37 71 71 71 F F F 8 160-179 37 37 37 71 71 71 F F F 9 180-199 36 36 36 69 69 69 F F F 10 200-219 32 32 32 62 62 62 T T T 11 220-239 29 29 29 58 58 58 T T T 12 240-259 34 34 34 64 64 64 T T T 13 260-279 32 32 32 61 61 61 F F F 14 280-299 38 38 38 72 72 72 F F F 15 300-319 38 38 38 72 72 72 F F F 16 320-339 36 36 36 69 69 69 T T T 17 340-359 32 32 32 61 61 61 T T T 18 360-379 36 36 36 68 68 68 T T T 19 380-399 35 35 35 67 67 67 F F F 20 400-419 33 33 33 63 63 63 F F F 21 420-439 34 34 34 65 65 65 F F F 22 440-459 34 34 34 64 64 64 F F F 23 460-479 38 38 38 72 72 72 T T T 24 480-499 34 34 34 64 64 64 T T T 25 500-519 30 30 30 59 59 59 F F F 26 520-539 37 37 37 71 71 71 F F F 27 540-559 37 37 37 71 71 71 F F F 28 560-579 34 34 34 65 65 65 T T T 29 580-599 36 36 36 68 68 68 T T T Max gen: 38, Max flush: 72 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 71 71 71 F F F 1 020-039 34 34 34 64 64 64 F F F 2 040-059 34 34 34 64 64 64 F F F 3 060-079 35 35 35 67 67 67 F F F 4 080-099 39 39 39 73 73 73 F F F 5 100-119 39 39 39 73 73 73 F F F 6 120-139 30 30 30 59 59 59 F F F 7 140-159 37 37 37 71 71 71 F F F 8 160-179 37 37 37 71 71 71 F F F 9 180-199 36 36 36 69 69 69 F F F 10 200-219 39 39 39 73 73 73 F F F 11 220-239 39 39 39 73 73 73 F F F 12 240-259 39 39 39 73 73 73 F F F 13 260-279 32 32 32 61 61 61 F F F 14 280-299 38 38 38 72 72 72 F F F 15 300-319 38 38 38 72 72 72 F F F 16 320-339 39 39 39 73 73 73 F F F 17 340-359 39 39 39 73 73 73 F F F 18 360-379 39 39 39 73 73 73 F F F 19 380-399 35 35 35 67 67 67 F F F 20 400-419 33 33 33 63 63 63 F F F 21 420-439 34 34 34 65 65 65 F F F 22 440-459 34 34 34 64 64 64 F F F 23 460-479 39 39 39 73 73 73 F F F 24 480-499 39 39 39 73 73 73 F F F 25 500-519 30 30 30 59 59 59 F F F 26 520-539 37 37 37 71 71 71 F F F 27 540-559 37 37 37 71 71 71 F F F 28 560-579 39 39 39 73 73 73 F F F 29 580-599 39 39 39 73 73 73 F F F Max gen: 39, Max flush: 73 Begin loop 20 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.369076099Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.412864129Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.415780732Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.415816493Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.415831023Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.415845254Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347} {"msg":"Crucible c406cd36-cf7d-4854-8e82-5d604e4f143e has session id: 357bbd05-6c6b-40c9-bb1e-438ca6623e09","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.416433396Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"Upstairs opts: Upstairs UUID: c406cd36-cf7d-4854-8e82-5d604e4f143e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.416488997Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"Crucible stats registered with UUID: c406cd36-cf7d-4854-8e82-5d604e4f143e","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.416519848Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.44704732Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.447079411Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347} {"msg":"c406cd36-cf7d-4854-8e82-5d604e4f143e active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.447114552Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.447131592Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"client":"0","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.447160313Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"client":"0","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.447188913Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"client":"1","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.447223304Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"client":"1","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.447261425Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"client":"2","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.447296236Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"client":"2","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.447329737Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"":"io task","client":"2","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.447378227Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"":"io task","client":"0","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.447414598Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"":"io task","client":"1","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"ds_connection connected from Ok(127.0.0.1:59588)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.447450539Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"":"io task","client":"2","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"ds_connection connected from Ok(127.0.0.1:44473)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.44748847Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"":"io task","client":"0","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"ds_connection connected from Ok(127.0.0.1:36643)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.447674474Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"":"io task","client":"1","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.448171315Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"client":"0","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.448213666Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"client":"0","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.448248856Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"client":"2","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.448281637Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"client":"2","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.448391709Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"client":"1","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.448437841Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"client":"1","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.44886127Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"client":"0","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.44889678Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"client":"2","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.448915481Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"client":"1","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.449425902Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"client":"1","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.449454283Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"client":"1","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.449470033Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.449483293Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"client":"2","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.449497193Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"client":"2","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.449510484Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.449522754Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"client":"0","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.449548614Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"client":"0","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"[0]R flush_numbers[0..12]: [71, 64, 64, 67, 73, 73, 59, 71, 71, 69, 73, 73]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.449563335Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"":"downstairs","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"[0]R generation[0..12]: [37, 34, 34, 35, 39, 39, 30, 37, 37, 36, 39, 39]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.449577175Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"":"downstairs","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.449591366Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"":"downstairs","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"[1]R flush_numbers[0..12]: [71, 64, 64, 67, 73, 73, 59, 71, 71, 69, 73, 73]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.449605056Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"":"downstairs","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"[1]R generation[0..12]: [37, 34, 34, 35, 39, 39, 30, 37, 37, 36, 39, 39]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.449618886Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"":"downstairs","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.449635467Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"":"downstairs","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"[2]R flush_numbers[0..12]: [71, 64, 64, 67, 73, 73, 59, 71, 71, 69, 73, 73]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.449649367Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"":"downstairs","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"[2]R generation[0..12]: [37, 34, 34, 35, 39, 39, 30, 37, 37, 36, 39, 39]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.449662827Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"":"downstairs","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.449676767Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.449698318Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"":"downstairs","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.449717138Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"":"downstairs","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"Next flush: 74","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.449734599Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"":"downstairs","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.449747939Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"":"downstairs","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.449761279Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.449773559Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"client":"0","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.44978688Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"client":"1","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.44980022Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"client":"2","":"downstairs","session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.44981361Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.449825911Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"c406cd36-cf7d-4854-8e82-5d604e4f143e is now active with session: 357bbd05-6c6b-40c9-bb1e-438ca6623e09","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.449838791Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.449853361Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347,"session_id":"357bbd05-6c6b-40c9-bb1e-438ca6623e09"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:36.449866241Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 560 len 16384 data: 3 2 2 2 02/30 Write block 441 len 36864 data: 8 4 4 4 4 4 4 2 2 03/30 Write block 583 len 28672 data: 4 4 4 5 4 4 4 04/30 Read block 298 len 4096 05/30 Read block 153 len 12288 06/30 Write block 242 len 36864 data: 4 5 4 5 6 6 6 4 5 07/30 Read block 222 len 16384 08/30 Flush 09/30 Write block 524 len 4096 data: 4 10/30 Read block 385 len 36864 11/30 Flush 12/30 Write block 312 len 32768 data: 3 3 3 3 2 2 2 3 13/30 Read block 508 len 20480 14/30 Write block 59 len 40960 data: 8 9 8 5 4 5 4 5 6 7 15/30 Read block 66 len 24576 16/30 Write block 94 len 12288 data: 5 7 5 17/30 Write block 368 len 40960 data: 3 3 4 5 6 4 4 5 5 6 18/30 Read block 33 len 20480 19/30 Write block 203 len 16384 data: 6 4 4 4 20/30 Read block 153 len 28672 21/30 Read block 239 len 20480 22/30 Read block 158 len 8192 23/30 Read block 83 len 28672 24/30 Write block 298 len 4096 data: 2 25/30 Read block 381 len 36864 26/30 Read block 242 len 36864 27/30 Read block 550 len 36864 28/30 Write block 264 len 4096 data: 2 29/30 Read block 388 len 32768 30/30 Read block 420 len 40960 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 8 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Write 10 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 19 18 18 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:51:36.470349266Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1347} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 71 71 71 F F F 1 020-039 34 34 34 64 64 64 F F F 2 040-059 34 34 34 64 64 64 T T T 3 060-079 35 35 35 67 67 67 T T T 4 080-099 39 39 39 73 73 73 T T T 5 100-119 39 39 39 73 73 73 F F F 6 120-139 30 30 30 59 59 59 F F F 7 140-159 37 37 37 71 71 71 F F F 8 160-179 37 37 37 71 71 71 F F F 9 180-199 36 36 36 69 69 69 F F F 10 200-219 39 39 39 73 73 73 T T T 11 220-239 39 39 39 73 73 73 F F F 12 240-259 40 40 40 74 74 74 F F F 13 260-279 32 32 32 61 61 61 T T T 14 280-299 38 38 38 72 72 72 T T T 15 300-319 38 38 38 72 72 72 T T T 16 320-339 39 39 39 73 73 73 F F F 17 340-359 39 39 39 73 73 73 F F F 18 360-379 39 39 39 73 73 73 T T T 19 380-399 35 35 35 67 67 67 F F F 20 400-419 33 33 33 63 63 63 F F F 21 420-439 34 34 34 65 65 65 F F F 22 440-459 40 40 40 74 74 74 F F F 23 460-479 39 39 39 73 73 73 F F F 24 480-499 39 39 39 73 73 73 F F F 25 500-519 30 30 30 59 59 59 F F F 26 520-539 40 40 40 75 75 75 F F F 27 540-559 37 37 37 71 71 71 F F F 28 560-579 40 40 40 74 74 74 F F F 29 580-599 40 40 40 74 74 74 F F F Max gen: 40, Max flush: 75 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 71 71 71 F F F 1 020-039 34 34 34 64 64 64 F F F 2 040-059 41 41 41 76 76 76 F F F 3 060-079 41 41 41 76 76 76 F F F 4 080-099 41 41 41 76 76 76 F F F 5 100-119 39 39 39 73 73 73 F F F 6 120-139 30 30 30 59 59 59 F F F 7 140-159 37 37 37 71 71 71 F F F 8 160-179 37 37 37 71 71 71 F F F 9 180-199 36 36 36 69 69 69 F F F 10 200-219 41 41 41 76 76 76 F F F 11 220-239 39 39 39 73 73 73 F F F 12 240-259 40 40 40 74 74 74 F F F 13 260-279 41 41 41 76 76 76 F F F 14 280-299 41 41 41 76 76 76 F F F 15 300-319 41 41 41 76 76 76 F F F 16 320-339 39 39 39 73 73 73 F F F 17 340-359 39 39 39 73 73 73 F F F 18 360-379 41 41 41 76 76 76 F F F 19 380-399 35 35 35 67 67 67 F F F 20 400-419 33 33 33 63 63 63 F F F 21 420-439 34 34 34 65 65 65 F F F 22 440-459 40 40 40 74 74 74 F F F 23 460-479 39 39 39 73 73 73 F F F 24 480-499 39 39 39 73 73 73 F F F 25 500-519 30 30 30 59 59 59 F F F 26 520-539 40 40 40 75 75 75 F F F 27 540-559 37 37 37 71 71 71 F F F 28 560-579 40 40 40 74 74 74 F F F 29 580-599 40 40 40 74 74 74 F F F Max gen: 41, Max flush: 76 Begin loop 21 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.749177157Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.793191581Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.793850225Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.793886246Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.793904787Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.793919807Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355} {"msg":"Crucible 1ec6af46-70a2-40b2-b80a-406fd1454e80 has session id: 15484b23-5117-4a7e-93d0-0337d1a7c6c1","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.794437498Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"Upstairs opts: Upstairs UUID: 1ec6af46-70a2-40b2-b80a-406fd1454e80, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.794471869Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"Crucible stats registered with UUID: 1ec6af46-70a2-40b2-b80a-406fd1454e80","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.794487489Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.825290227Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.825316878Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355} {"msg":"1ec6af46-70a2-40b2-b80a-406fd1454e80 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.82539435Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.82541211Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"client":"0","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.82543582Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"client":"0","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.825449931Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"client":"1","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.825463941Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"client":"1","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.825477681Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"client":"2","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.825491312Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"client":"2","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.825508232Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"":"io task","client":"2","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.825523212Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"":"io task","client":"0","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.825558333Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"":"io task","client":"1","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"ds_connection connected from Ok(127.0.0.1:38885)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.825706016Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"":"io task","client":"1","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"ds_connection connected from Ok(127.0.0.1:59786)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.82585174Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"":"io task","client":"2","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"ds_connection connected from Ok(127.0.0.1:58401)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.82589056Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"":"io task","client":"0","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.826563485Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"client":"2","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.826594116Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"client":"2","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.826610956Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"client":"0","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.826625366Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"client":"0","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.826649197Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"client":"1","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.826678027Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"client":"1","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.827290871Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"client":"1","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.827321321Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"client":"2","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.827347592Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"client":"0","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.827833563Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"client":"1","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.827862603Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"client":"1","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.827877604Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.827969046Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"client":"0","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.827998846Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"client":"0","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.828015347Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.828029527Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"client":"2","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.828044387Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"client":"2","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"[0]R flush_numbers[0..12]: [71, 64, 76, 76, 76, 73, 59, 71, 71, 69, 76, 73]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.828059597Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"":"downstairs","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"[0]R generation[0..12]: [37, 34, 41, 41, 41, 39, 30, 37, 37, 36, 41, 39]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.828074628Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"":"downstairs","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.828089718Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"":"downstairs","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"[1]R flush_numbers[0..12]: [71, 64, 76, 76, 76, 73, 59, 71, 71, 69, 76, 73]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.828104978Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"":"downstairs","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"[1]R generation[0..12]: [37, 34, 41, 41, 41, 39, 30, 37, 37, 36, 41, 39]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.828119509Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"":"downstairs","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.828134529Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"":"downstairs","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"[2]R flush_numbers[0..12]: [71, 64, 76, 76, 76, 73, 59, 71, 71, 69, 76, 73]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.828152959Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"":"downstairs","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"[2]R generation[0..12]: [37, 34, 41, 41, 41, 39, 30, 37, 37, 36, 41, 39]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.82816829Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"":"downstairs","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:51:40.82818456Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"":"downstairs","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.82820542Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"":"downstairs","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.828219521Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"":"downstairs","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"Next flush: 77","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.828233911Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"":"downstairs","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.828248241Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"":"downstairs","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.828262802Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.828276162Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"client":"0","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.828290702Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"client":"1","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.828305123Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"client":"2","":"downstairs","session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.828319603Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.828333243Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"1ec6af46-70a2-40b2-b80a-406fd1454e80 is now active with session: 15484b23-5117-4a7e-93d0-0337d1a7c6c1","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-28T04:51:40.828347064Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.828362974Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355,"session_id":"15484b23-5117-4a7e-93d0-0337d1a7c6c1"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:40.828376874Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 13 len 24576 data: 4 4 6 6 6 5 03/30 Read block 3 len 32768 04/30 Read block 575 len 8192 05/30 Write block 537 len 12288 data: 5 6 5 06/30 Flush 07/30 Write block 369 len 4096 data: 4 08/30 Write block 54 len 32768 data: 7 7 9 9 9 9 10 9 09/30 Read block 210 len 4096 10/30 Write block 134 len 36864 data: 4 4 4 3 4 4 4 4 4 11/30 Write block 588 len 12288 data: 5 5 5 12/30 Write block 249 len 36864 data: 5 6 5 5 5 5 4 4 5 13/30 Write block 300 len 16384 data: 3 4 4 4 14/30 Read block 459 len 40960 15/30 Read block 188 len 16384 16/30 Read block 213 len 36864 17/30 Flush 18/30 Write block 154 len 32768 data: 3 3 3 3 3 3 3 4 19/30 Read block 284 len 4096 20/30 Flush 21/30 Write block 26 len 16384 data: 4 4 4 4 22/30 Flush 23/30 Write block 265 len 32768 data: 2 2 2 3 6 7 6 6 24/30 Read block 252 len 4096 25/30 Flush 26/30 Write block 503 len 4096 data: 5 27/30 Write block 87 len 20480 data: 3 3 5 4 4 28/30 Write block 230 len 24576 data: 2 2 2 4 5 5 29/30 Read block 440 len 28672 30/30 Write block 279 len 4096 data: 3 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 1 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 6 Done Done Sent false 1028 Acked Read 7 Done Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 3 6 Done 4 3 2 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:51:40.846572539Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1355} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 78 78 78 F F F 1 020-039 42 42 42 81 81 81 F F F 2 040-059 42 42 42 79 79 79 F F F 3 060-079 42 42 42 79 79 79 F F F 4 080-099 41 41 41 76 76 76 T T T 5 100-119 39 39 39 73 73 73 F F F 6 120-139 42 42 42 79 79 79 F F F 7 140-159 42 42 42 80 80 80 F F F 8 160-179 42 42 42 80 80 80 F F F 9 180-199 36 36 36 69 69 69 F F F 10 200-219 41 41 41 76 76 76 F F F 11 220-239 39 39 39 73 73 73 T T T 12 240-259 42 42 42 79 79 79 F F F 13 260-279 42 42 42 82 82 82 T T T 14 280-299 41 41 41 76 76 76 F F F 15 300-319 42 42 42 79 79 79 F F F 16 320-339 39 39 39 73 73 73 F F F 17 340-359 39 39 39 73 73 73 F F F 18 360-379 42 42 42 79 79 79 F F F 19 380-399 35 35 35 67 67 67 F F F 20 400-419 33 33 33 63 63 63 F F F 21 420-439 34 34 34 65 65 65 F F F 22 440-459 40 40 40 74 74 74 F F F 23 460-479 39 39 39 73 73 73 F F F 24 480-499 39 39 39 73 73 73 F F F 25 500-519 30 30 30 59 59 59 T T T 26 520-539 42 42 42 78 78 78 F F F 27 540-559 37 37 37 71 71 71 F F F 28 560-579 40 40 40 74 74 74 F F F 29 580-599 42 42 42 79 79 79 F F F Max gen: 42, Max flush: 82 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 78 78 78 F F F 1 020-039 42 42 42 81 81 81 F F F 2 040-059 42 42 42 79 79 79 F F F 3 060-079 42 42 42 79 79 79 F F F 4 080-099 43 43 43 83 83 83 F F F 5 100-119 39 39 39 73 73 73 F F F 6 120-139 42 42 42 79 79 79 F F F 7 140-159 42 42 42 80 80 80 F F F 8 160-179 42 42 42 80 80 80 F F F 9 180-199 36 36 36 69 69 69 F F F 10 200-219 41 41 41 76 76 76 F F F 11 220-239 43 43 43 83 83 83 F F F 12 240-259 42 42 42 79 79 79 F F F 13 260-279 43 43 43 83 83 83 F F F 14 280-299 41 41 41 76 76 76 F F F 15 300-319 42 42 42 79 79 79 F F F 16 320-339 39 39 39 73 73 73 F F F 17 340-359 39 39 39 73 73 73 F F F 18 360-379 42 42 42 79 79 79 F F F 19 380-399 35 35 35 67 67 67 F F F 20 400-419 33 33 33 63 63 63 F F F 21 420-439 34 34 34 65 65 65 F F F 22 440-459 40 40 40 74 74 74 F F F 23 460-479 39 39 39 73 73 73 F F F 24 480-499 39 39 39 73 73 73 F F F 25 500-519 43 43 43 83 83 83 F F F 26 520-539 42 42 42 78 78 78 F F F 27 540-559 37 37 37 71 71 71 F F F 28 560-579 40 40 40 74 74 74 F F F 29 580-599 42 42 42 79 79 79 F F F Max gen: 43, Max flush: 83 Begin loop 22 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.101270836Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:51:45.144821611Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.14522092Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.177706725Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.178270497Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.178307767Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.178323368Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.178339748Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363} {"msg":"Crucible 4d26147f-32e3-472d-8a03-529a378d8465 has session id: cc4b883c-4963-4c7a-a339-7755fe3e8321","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.178780458Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"Upstairs opts: Upstairs UUID: 4d26147f-32e3-472d-8a03-529a378d8465, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.178823599Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"Crucible stats registered with UUID: 4d26147f-32e3-472d-8a03-529a378d8465","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.178845309Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.20975762Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.20978603Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363} {"msg":"4d26147f-32e3-472d-8a03-529a378d8465 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.209862962Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.209881272Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"client":"0","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.209905773Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"client":"0","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.209920583Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"client":"1","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.209934273Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"client":"1","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.209947554Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"client":"2","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.209967344Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"client":"2","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.209993325Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"":"io task","client":"2","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.210021695Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"":"io task","client":"0","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.210050756Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"":"io task","client":"1","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"ds_connection connected from Ok(127.0.0.1:54904)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.210100837Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"":"io task","client":"2","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"ds_connection connected from Ok(127.0.0.1:52240)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.210309962Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"":"io task","client":"0","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"ds_connection connected from Ok(127.0.0.1:47326)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.210354163Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"":"io task","client":"1","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.210954056Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"client":"2","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.211036208Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"client":"2","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.211107019Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"client":"1","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.21113793Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"client":"1","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.211193011Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"client":"0","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.211223741Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"client":"0","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.211542099Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"client":"2","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.211662511Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"client":"0","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.211693612Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"client":"1","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.212122821Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"client":"2","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.212154502Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"client":"2","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.212169622Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.212201243Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"client":"1","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.212215813Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"client":"1","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.212229533Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.212243304Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"client":"0","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.212256854Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"client":"0","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"[0]R flush_numbers[0..12]: [78, 81, 79, 79, 83, 73, 79, 80, 80, 69, 76, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.212270584Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"":"downstairs","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"[0]R generation[0..12]: [42, 42, 42, 42, 43, 39, 42, 42, 42, 36, 41, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.212284255Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"":"downstairs","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.212299305Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"":"downstairs","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"[1]R flush_numbers[0..12]: [78, 81, 79, 79, 83, 73, 79, 80, 80, 69, 76, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.212313135Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"":"downstairs","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"[1]R generation[0..12]: [42, 42, 42, 42, 43, 39, 42, 42, 42, 36, 41, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.212326885Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"":"downstairs","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.212340896Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"":"downstairs","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"[2]R flush_numbers[0..12]: [78, 81, 79, 79, 83, 73, 79, 80, 80, 69, 76, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.212354606Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"":"downstairs","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"[2]R generation[0..12]: [42, 42, 42, 42, 43, 39, 42, 42, 42, 36, 41, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.212368216Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"":"downstairs","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.212382317Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"":"downstairs","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.212397237Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"":"downstairs","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.212410737Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"":"downstairs","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"Next flush: 84","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.212423887Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"":"downstairs","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.212437198Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"":"downstairs","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:51:45.212450568Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.212470928Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"client":"0","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.212484339Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"client":"1","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.212497599Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"client":"2","":"downstairs","session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.2125109Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.21252346Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"4d26147f-32e3-472d-8a03-529a378d8465 is now active with session: cc4b883c-4963-4c7a-a339-7755fe3e8321","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.21253657Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.21254861Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363,"session_id":"cc4b883c-4963-4c7a-a339-7755fe3e8321"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:45.21256089Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 165 len 32768 data: 2 2 2 4 4 4 5 4 02/30 Read block 263 len 28672 03/30 Write block 195 len 4096 data: 7 04/30 Flush 05/30 Write block 210 len 28672 data: 7 7 7 6 5 5 5 06/30 Flush 07/30 Write block 194 len 36864 data: 7 8 6 6 6 6 7 6 7 08/30 Write block 393 len 40960 data: 3 3 5 5 4 5 6 5 5 4 09/30 Read block 111 len 4096 10/30 Write block 46 len 4096 data: 5 11/30 Write block 323 len 4096 data: 3 12/30 Read block 489 len 36864 13/30 Read block 511 len 12288 14/30 Read block 123 len 20480 15/30 Read block 287 len 12288 16/30 Write block 253 len 36864 data: 6 6 5 5 6 5 4 3 3 17/30 Write block 348 len 16384 data: 5 4 5 6 18/30 Read block 278 len 28672 19/30 Flush 20/30 Write block 270 len 36864 data: 8 7 7 6 6 4 4 5 5 21/30 Write block 548 len 20480 data: 4 4 5 5 6 22/30 Read block 563 len 4096 23/30 Flush 24/30 Write block 270 len 24576 data: 9 8 8 7 7 5 25/30 Read block 506 len 28672 26/30 Write block 476 len 28672 data: 6 5 5 4 5 5 5 27/30 Read block 141 len 20480 28/30 Write block 590 len 16384 data: 6 4 4 3 29/30 Write block 73 len 12288 data: 7 6 6 30/30 Read block 189 len 40960 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 6 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Read 10 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 6 7 7 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:51:45.232541664Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1363} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 78 78 78 F F F 1 020-039 42 42 42 81 81 81 F F F 2 040-059 44 44 44 86 86 86 F F F 3 060-079 42 42 42 79 79 79 T T T 4 080-099 43 43 43 83 83 83 F F F 5 100-119 39 39 39 73 73 73 F F F 6 120-139 42 42 42 79 79 79 F F F 7 140-159 42 42 42 80 80 80 F F F 8 160-179 44 44 44 84 84 84 F F F 9 180-199 44 44 44 86 86 86 F F F 10 200-219 44 44 44 86 86 86 F F F 11 220-239 43 43 43 83 83 83 F F F 12 240-259 44 44 44 86 86 86 F F F 13 260-279 44 44 44 87 87 87 T T T 14 280-299 41 41 41 76 76 76 F F F 15 300-319 42 42 42 79 79 79 F F F 16 320-339 44 44 44 86 86 86 F F F 17 340-359 44 44 44 86 86 86 F F F 18 360-379 42 42 42 79 79 79 F F F 19 380-399 44 44 44 86 86 86 F F F 20 400-419 44 44 44 86 86 86 F F F 21 420-439 34 34 34 65 65 65 F F F 22 440-459 40 40 40 74 74 74 F F F 23 460-479 39 39 39 73 73 73 T T T 24 480-499 39 39 39 73 73 73 T T T 25 500-519 43 43 43 83 83 83 F F F 26 520-539 42 42 42 78 78 78 F F F 27 540-559 44 44 44 87 87 87 F F F 28 560-579 40 40 40 74 74 74 F F F 29 580-599 42 42 42 79 79 79 T T T Max gen: 44, Max flush: 87 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 78 78 78 F F F 1 020-039 42 42 42 81 81 81 F F F 2 040-059 44 44 44 86 86 86 F F F 3 060-079 45 45 45 88 88 88 F F F 4 080-099 43 43 43 83 83 83 F F F 5 100-119 39 39 39 73 73 73 F F F 6 120-139 42 42 42 79 79 79 F F F 7 140-159 42 42 42 80 80 80 F F F 8 160-179 44 44 44 84 84 84 F F F 9 180-199 44 44 44 86 86 86 F F F 10 200-219 44 44 44 86 86 86 F F F 11 220-239 43 43 43 83 83 83 F F F 12 240-259 44 44 44 86 86 86 F F F 13 260-279 45 45 45 88 88 88 F F F 14 280-299 41 41 41 76 76 76 F F F 15 300-319 42 42 42 79 79 79 F F F 16 320-339 44 44 44 86 86 86 F F F 17 340-359 44 44 44 86 86 86 F F F 18 360-379 42 42 42 79 79 79 F F F 19 380-399 44 44 44 86 86 86 F F F 20 400-419 44 44 44 86 86 86 F F F 21 420-439 34 34 34 65 65 65 F F F 22 440-459 40 40 40 74 74 74 F F F 23 460-479 45 45 45 88 88 88 F F F 24 480-499 45 45 45 88 88 88 F F F 25 500-519 43 43 43 83 83 83 F F F 26 520-539 42 42 42 78 78 78 F F F 27 540-559 44 44 44 87 87 87 F F F 28 560-579 40 40 40 74 74 74 F F F 29 580-599 45 45 45 88 88 88 F F F Max gen: 45, Max flush: 88 Begin loop 23 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.530701464Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.574169327Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.57473645Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.57477172Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.574785991Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.574801041Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371} {"msg":"Crucible 16c2649e-b9d9-41fc-bd73-14d97232b15f has session id: d64d2c07-60cf-4175-b746-4e359dc7b84f","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.575151648Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"Upstairs opts: Upstairs UUID: 16c2649e-b9d9-41fc-bd73-14d97232b15f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.575198Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"Crucible stats registered with UUID: 16c2649e-b9d9-41fc-bd73-14d97232b15f","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.57522322Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.60609481Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.60612842Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371} {"msg":"16c2649e-b9d9-41fc-bd73-14d97232b15f active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.606160581Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.606179302Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"client":"0","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.606208462Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"client":"0","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.606226233Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"client":"1","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.606243023Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"client":"1","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.606259373Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"client":"2","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.606276164Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"client":"2","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.606292464Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"":"io task","client":"2","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.606320505Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"":"io task","client":"0","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.606354776Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"":"io task","client":"1","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"ds_connection connected from Ok(127.0.0.1:43689)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.606418217Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"":"io task","client":"2","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"ds_connection connected from Ok(127.0.0.1:41162)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.606458888Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"":"io task","client":"1","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"ds_connection connected from Ok(127.0.0.1:64438)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.60657544Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"":"io task","client":"0","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.607197654Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"client":"0","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.607230684Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"client":"0","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.607247805Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"client":"1","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.607262035Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"client":"1","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.607333367Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"client":"2","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.607359477Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"client":"2","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.607896339Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"client":"0","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.608026202Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"client":"1","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.608051612Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"client":"2","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.60842272Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"client":"0","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.608453881Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"client":"0","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.608469691Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.608631115Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"client":"1","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.608659015Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"client":"1","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.608673886Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.608686366Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"client":"2","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.608703316Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"client":"2","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"[0]R flush_numbers[0..12]: [78, 81, 86, 88, 83, 73, 79, 80, 84, 86, 86, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.608717567Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"":"downstairs","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"[0]R generation[0..12]: [42, 42, 44, 45, 43, 39, 42, 42, 44, 44, 44, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.608731757Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"":"downstairs","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.608745977Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"":"downstairs","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"[1]R flush_numbers[0..12]: [78, 81, 86, 88, 83, 73, 79, 80, 84, 86, 86, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.608759757Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"":"downstairs","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"[1]R generation[0..12]: [42, 42, 44, 45, 43, 39, 42, 42, 44, 44, 44, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.608773398Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"":"downstairs","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.608787518Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"":"downstairs","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"[2]R flush_numbers[0..12]: [78, 81, 86, 88, 83, 73, 79, 80, 84, 86, 86, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.608801198Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"":"downstairs","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"[2]R generation[0..12]: [42, 42, 44, 45, 43, 39, 42, 42, 44, 44, 44, 43]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:51:49.608814599Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"":"downstairs","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.60889202Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"":"downstairs","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.608915031Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"":"downstairs","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.608929311Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"":"downstairs","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"Next flush: 89","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.608942732Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"":"downstairs","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.608956092Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"":"downstairs","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.608969372Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.608986073Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"client":"0","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.609000183Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"client":"1","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.609013513Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"client":"2","":"downstairs","session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.609026923Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.609039274Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"16c2649e-b9d9-41fc-bd73-14d97232b15f is now active with session: d64d2c07-60cf-4175-b746-4e359dc7b84f","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.609052044Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.609067944Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371,"session_id":"d64d2c07-60cf-4175-b746-4e359dc7b84f"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:49.609080764Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 164 len 4096 data: 4 02/30 Read block 390 len 36864 03/30 Write block 399 len 24576 data: 7 6 6 5 4 4 04/30 Read block 562 len 16384 05/30 Read block 430 len 8192 06/30 Read block 460 len 28672 07/30 Write block 512 len 40960 data: 6 6 6 5 4 5 5 6 5 5 08/30 Read block 522 len 28672 09/30 Read block 532 len 36864 10/30 Write block 249 len 12288 data: 6 7 6 11/30 Read block 2 len 16384 12/30 Read block 2 len 24576 13/30 Read block 99 len 20480 14/30 Read block 407 len 4096 15/30 Write block 273 len 24576 data: 8 8 6 5 6 6 16/30 Write block 578 len 28672 data: 5 5 4 3 2 5 5 17/30 Read block 372 len 8192 18/30 Read block 391 len 40960 19/30 Read block 114 len 24576 20/30 Read block 452 len 36864 21/30 Read block 39 len 4096 22/30 Read block 357 len 8192 23/30 Read block 390 len 36864 24/30 Read block 26 len 20480 25/30 Read block 107 len 40960 26/30 Write block 591 len 8192 data: 5 5 27/30 Read block 106 len 4096 28/30 Write block 492 len 4096 data: 4 29/30 Write block 334 len 32768 data: 7 8 8 6 7 6 6 4 30/30 Read block 185 len 16384 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Write 6 Done Done Done false 1003 Acked Read 4 Done Done Done false 1004 Acked Read 2 Done Done Done false 1005 Acked Read 7 Done Done Done false 1006 Acked Write 10 Done Done Done false 1007 Acked Read 7 Done Done Done false 1008 Acked Read 9 Done Done Done false 1009 Acked Write 3 Done Done Done false 1010 Acked Read 4 Done Done Done false 1011 Acked Read 6 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 4 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 30 30 29 89 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:51:49.628811952Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1371} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 78 78 78 F F F 1 020-039 42 42 42 81 81 81 F F F 2 040-059 44 44 44 86 86 86 F F F 3 060-079 45 45 45 88 88 88 F F F 4 080-099 43 43 43 83 83 83 F F F 5 100-119 39 39 39 73 73 73 F F F 6 120-139 42 42 42 79 79 79 F F F 7 140-159 42 42 42 80 80 80 F F F 8 160-179 44 44 44 84 84 84 T T T 9 180-199 44 44 44 86 86 86 F F F 10 200-219 44 44 44 86 86 86 F F F 11 220-239 43 43 43 83 83 83 F F F 12 240-259 44 44 44 86 86 86 T T T 13 260-279 45 45 45 88 88 88 T T T 14 280-299 41 41 41 76 76 76 F F F 15 300-319 42 42 42 79 79 79 F F F 16 320-339 44 44 44 86 86 86 T T T 17 340-359 44 44 44 86 86 86 T T T 18 360-379 42 42 42 79 79 79 F F F 19 380-399 44 44 44 86 86 86 T T T 20 400-419 44 44 44 86 86 86 T T T 21 420-439 34 34 34 65 65 65 F F F 22 440-459 40 40 40 74 74 74 F F F 23 460-479 45 45 45 88 88 88 F F F 24 480-499 45 45 45 88 88 88 T T T 25 500-519 43 43 43 83 83 83 T T T 26 520-539 42 42 42 78 78 78 T T T 27 540-559 44 44 44 87 87 87 F F F 28 560-579 40 40 40 74 74 74 T T T 29 580-599 45 45 45 88 88 88 T T T Max gen: 45, Max flush: 88 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 78 78 78 F F F 1 020-039 42 42 42 81 81 81 F F F 2 040-059 44 44 44 86 86 86 F F F 3 060-079 45 45 45 88 88 88 F F F 4 080-099 43 43 43 83 83 83 F F F 5 100-119 39 39 39 73 73 73 F F F 6 120-139 42 42 42 79 79 79 F F F 7 140-159 42 42 42 80 80 80 F F F 8 160-179 46 46 46 89 89 89 F F F 9 180-199 44 44 44 86 86 86 F F F 10 200-219 44 44 44 86 86 86 F F F 11 220-239 43 43 43 83 83 83 F F F 12 240-259 46 46 46 89 89 89 F F F 13 260-279 46 46 46 89 89 89 F F F 14 280-299 41 41 41 76 76 76 F F F 15 300-319 42 42 42 79 79 79 F F F 16 320-339 46 46 46 89 89 89 F F F 17 340-359 46 46 46 89 89 89 F F F 18 360-379 42 42 42 79 79 79 F F F 19 380-399 46 46 46 89 89 89 F F F 20 400-419 46 46 46 89 89 89 F F F 21 420-439 34 34 34 65 65 65 F F F 22 440-459 40 40 40 74 74 74 F F F 23 460-479 45 45 45 88 88 88 F F F 24 480-499 46 46 46 89 89 89 F F F 25 500-519 46 46 46 89 89 89 F F F 26 520-539 46 46 46 89 89 89 F F F 27 540-559 44 44 44 87 87 87 F F F 28 560-579 46 46 46 89 89 89 F F F 29 580-599 46 46 46 89 89 89 F F F Max gen: 46, Max flush: 89 Begin loop 24 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:53.929921766Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:53.973591834Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:53.974165436Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:53.974202267Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:53.974217787Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:53.974232157Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379} {"msg":"Crucible eac09271-4fc6-42a6-a20f-f5a76235adc7 has session id: 8199cec5-6a80-4af5-9b08-c1f89f928007","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:53.974712288Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"Upstairs opts: Upstairs UUID: eac09271-4fc6-42a6-a20f-f5a76235adc7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:53.974751829Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"Crucible stats registered with UUID: eac09271-4fc6-42a6-a20f-f5a76235adc7","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:53.974768399Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.005935715Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.005982166Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379} {"msg":"eac09271-4fc6-42a6-a20f-f5a76235adc7 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.005998877Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.006015737Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"client":"0","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.006044758Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"client":"0","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.006073918Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"client":"1","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.006108019Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"client":"1","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.00614157Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"client":"2","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.00617643Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"client":"2","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.006211341Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"":"io task","client":"2","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.006247652Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"":"io task","client":"0","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.006282003Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"":"io task","client":"1","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"ds_connection connected from Ok(127.0.0.1:56433)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.006316624Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"":"io task","client":"0","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"ds_connection connected from Ok(127.0.0.1:62290)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.006350004Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"":"io task","client":"2","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"ds_connection connected from Ok(127.0.0.1:42248)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.006382965Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"":"io task","client":"1","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.007146111Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"client":"0","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.007194163Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"client":"0","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.007228623Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"client":"1","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.007261244Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"client":"1","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.007291445Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"client":"2","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.007322385Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"client":"2","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.007729404Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"client":"1","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.007761555Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"client":"0","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.007897188Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"client":"2","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.008321157Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"client":"0","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.008348428Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"client":"0","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.008362828Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.008424799Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"client":"1","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.00845023Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"client":"1","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.00846486Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.008484341Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"client":"2","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.008501401Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"client":"2","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"[0]R flush_numbers[0..12]: [78, 81, 86, 88, 83, 73, 79, 80, 89, 86, 86, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.008528541Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"":"downstairs","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"[0]R generation[0..12]: [42, 42, 44, 45, 43, 39, 42, 42, 46, 44, 44, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.008543182Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"":"downstairs","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.008557462Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"":"downstairs","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"[1]R flush_numbers[0..12]: [78, 81, 86, 88, 83, 73, 79, 80, 89, 86, 86, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.008571282Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"":"downstairs","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"[1]R generation[0..12]: [42, 42, 44, 45, 43, 39, 42, 42, 46, 44, 44, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.008584863Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"":"downstairs","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.008598833Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"":"downstairs","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"[2]R flush_numbers[0..12]: [78, 81, 86, 88, 83, 73, 79, 80, 89, 86, 86, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.008612573Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"":"downstairs","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"[2]R generation[0..12]: [42, 42, 44, 45, 43, 39, 42, 42, 46, 44, 44, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.008626244Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"":"downstairs","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.008640244Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"":"downstairs","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"Max found gen is 47","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.008653264Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"":"downstairs","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"Generation requested: 48 >= found:47","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.008666524Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"":"downstairs","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"Next flush: 90","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.008679585Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"":"downstairs","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.008692755Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"":"downstairs","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:51:54.008706125Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.008728366Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"client":"0","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.008741746Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"client":"1","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.008754926Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"client":"2","":"downstairs","session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.008768207Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.008780557Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"eac09271-4fc6-42a6-a20f-f5a76235adc7 is now active with session: 8199cec5-6a80-4af5-9b08-c1f89f928007","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.008793437Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.008805547Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379,"session_id":"8199cec5-6a80-4af5-9b08-c1f89f928007"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:54.008818018Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 402 len 20480 data: 6 5 5 4 4 02/30 Read block 414 len 28672 03/30 Read block 25 len 24576 04/30 Write block 500 len 20480 data: 6 5 4 6 4 05/30 Write block 408 len 12288 data: 4 4 3 06/30 Read block 120 len 40960 07/30 Write block 155 len 24576 data: 4 4 4 4 4 4 08/30 Write block 123 len 24576 data: 7 5 3 3 3 4 09/30 Read block 112 len 28672 10/30 Write block 314 len 8192 data: 4 4 11/30 Read block 506 len 4096 12/30 Read block 358 len 32768 13/30 Write block 147 len 24576 data: 5 5 5 4 4 5 14/30 Read block 1 len 8192 15/30 Write block 208 len 28672 data: 5 4 8 8 8 7 6 16/30 Write block 105 len 20480 data: 3 2 3 2 2 17/30 Write block 25 len 8192 data: 5 5 18/30 Read block 12 len 32768 19/30 Write block 184 len 20480 data: 6 6 8 8 8 20/30 Read block 162 len 32768 21/30 Write block 313 len 16384 data: 4 5 5 3 22/30 Write block 46 len 4096 data: 6 23/30 Read block 426 len 4096 24/30 Write block 585 len 4096 data: 5 25/30 Read block 63 len 20480 26/30 Read block 348 len 32768 27/30 Write block 65 len 40960 data: 5 6 7 8 8 8 9 8 8 7 28/30 Read block 3 len 12288 29/30 Read block 294 len 16384 30/30 Read block 15 len 4096 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Read 6 Done Done Done false 1003 Acked Write 5 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Read 10 Done Done Done false 1006 Acked Write 6 Done Done Done false 1007 Acked Write 6 Done Done Done false 1008 Acked Read 7 Done Done Done false 1009 Acked Write 2 Done Done Done false 1010 Acked Read 1 Done Done Done false 1011 Acked Read 8 Done Done Done false 1012 Acked Write 6 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Write 7 Done Done Done false 1015 Acked Write 5 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:51:54.027032743Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1379} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 78 78 78 F F F 1 020-039 42 42 42 81 81 81 T T T 2 040-059 44 44 44 86 86 86 T T T 3 060-079 45 45 45 88 88 88 T T T 4 080-099 43 43 43 83 83 83 F F F 5 100-119 39 39 39 73 73 73 T T T 6 120-139 42 42 42 79 79 79 T T T 7 140-159 42 42 42 80 80 80 T T T 8 160-179 46 46 46 89 89 89 T T T 9 180-199 44 44 44 86 86 86 T T T 10 200-219 44 44 44 86 86 86 T T T 11 220-239 43 43 43 83 83 83 F F F 12 240-259 46 46 46 89 89 89 F F F 13 260-279 46 46 46 89 89 89 F F F 14 280-299 41 41 41 76 76 76 F F F 15 300-319 42 42 42 79 79 79 T T T 16 320-339 46 46 46 89 89 89 F F F 17 340-359 46 46 46 89 89 89 F F F 18 360-379 42 42 42 79 79 79 F F F 19 380-399 46 46 46 89 89 89 F F F 20 400-419 46 46 46 89 89 89 T T T 21 420-439 34 34 34 65 65 65 F F F 22 440-459 40 40 40 74 74 74 F F F 23 460-479 45 45 45 88 88 88 F F F 24 480-499 46 46 46 89 89 89 F F F 25 500-519 46 46 46 89 89 89 T T T 26 520-539 46 46 46 89 89 89 F F F 27 540-559 44 44 44 87 87 87 F F F 28 560-579 46 46 46 89 89 89 F F F 29 580-599 46 46 46 89 89 89 T T T Max gen: 46, Max flush: 89 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 78 78 78 F F F 1 020-039 47 47 47 90 90 90 F F F 2 040-059 47 47 47 90 90 90 F F F 3 060-079 47 47 47 90 90 90 F F F 4 080-099 43 43 43 83 83 83 F F F 5 100-119 47 47 47 90 90 90 F F F 6 120-139 47 47 47 90 90 90 F F F 7 140-159 47 47 47 90 90 90 F F F 8 160-179 47 47 47 90 90 90 F F F 9 180-199 47 47 47 90 90 90 F F F 10 200-219 47 47 47 90 90 90 F F F 11 220-239 43 43 43 83 83 83 F F F 12 240-259 46 46 46 89 89 89 F F F 13 260-279 46 46 46 89 89 89 F F F 14 280-299 41 41 41 76 76 76 F F F 15 300-319 47 47 47 90 90 90 F F F 16 320-339 46 46 46 89 89 89 F F F 17 340-359 46 46 46 89 89 89 F F F 18 360-379 42 42 42 79 79 79 F F F 19 380-399 46 46 46 89 89 89 F F F 20 400-419 47 47 47 90 90 90 F F F 21 420-439 34 34 34 65 65 65 F F F 22 440-459 40 40 40 74 74 74 F F F 23 460-479 45 45 45 88 88 88 F F F 24 480-499 46 46 46 89 89 89 F F F 25 500-519 47 47 47 90 90 90 F F F 26 520-539 46 46 46 89 89 89 F F F 27 540-559 44 44 44 87 87 87 F F F 28 560-579 46 46 46 89 89 89 F F F 29 580-599 47 47 47 90 90 90 F F F Max gen: 47, Max flush: 90 Begin loop 25 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.334518105Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:51:58.377083509Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.377477467Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.409111054Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.409688196Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.409724727Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.409739227Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.409753897Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387} {"msg":"Crucible 7bc4a9d8-1f94-42e0-9c60-42e8cf866daf has session id: 23da663b-503f-46aa-8af7-ad49c11b1847","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.410249808Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"Upstairs opts: Upstairs UUID: 7bc4a9d8-1f94-42e0-9c60-42e8cf866daf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.410297669Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"Crucible stats registered with UUID: 7bc4a9d8-1f94-42e0-9c60-42e8cf866daf","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.4103229Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.441775142Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.441822893Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387} {"msg":"7bc4a9d8-1f94-42e0-9c60-42e8cf866daf active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.441854954Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.441885814Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"client":"0","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.441934575Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"client":"0","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.441972956Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"client":"1","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.442008697Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"client":"1","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.442045608Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"client":"2","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.442072578Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"client":"2","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.442091779Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"":"io task","client":"2","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.44211431Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"":"io task","client":"0","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.44214741Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"":"io task","client":"1","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"ds_connection connected from Ok(127.0.0.1:60511)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.442183711Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"":"io task","client":"0","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"ds_connection connected from Ok(127.0.0.1:38486)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.442327734Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"":"io task","client":"2","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"ds_connection connected from Ok(127.0.0.1:61064)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.442377165Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"":"io task","client":"1","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.443029629Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"client":"1","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.44306118Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"client":"1","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.44307828Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"client":"2","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.443093351Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"client":"2","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.443108371Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"client":"0","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.443123651Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"client":"0","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.443646813Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"client":"1","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.443686754Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"client":"2","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.443803966Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"client":"0","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.444245866Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"client":"1","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.444271806Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"client":"1","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.444287407Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.444301257Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"client":"2","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.444315877Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"client":"2","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.444334038Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.444348198Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"client":"0","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.444362938Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"client":"0","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"[0]R flush_numbers[0..12]: [78, 90, 90, 90, 83, 90, 90, 90, 90, 90, 90, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.444384009Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"":"downstairs","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"[0]R generation[0..12]: [42, 47, 47, 47, 43, 47, 47, 47, 47, 47, 47, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.444399359Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"":"downstairs","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.444415189Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"":"downstairs","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"[1]R flush_numbers[0..12]: [78, 90, 90, 90, 83, 90, 90, 90, 90, 90, 90, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.44442996Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"":"downstairs","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"[1]R generation[0..12]: [42, 47, 47, 47, 43, 47, 47, 47, 47, 47, 47, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.44444491Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"":"downstairs","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.44445966Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"":"downstairs","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"[2]R flush_numbers[0..12]: [78, 90, 90, 90, 83, 90, 90, 90, 90, 90, 90, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.444474521Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"":"downstairs","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"[2]R generation[0..12]: [42, 47, 47, 47, 43, 47, 47, 47, 47, 47, 47, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.444489471Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"":"downstairs","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.444504871Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"":"downstairs","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.444519112Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"":"downstairs","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"Generation requested: 50 >= found:48","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.444533552Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"":"downstairs","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"Next flush: 91","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:51:58.444550342Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"":"downstairs","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.444640724Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"":"downstairs","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.444672935Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.444696415Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"client":"0","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.444711996Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"client":"1","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.444726476Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"client":"2","":"downstairs","session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.444739936Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.444756207Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"7bc4a9d8-1f94-42e0-9c60-42e8cf866daf is now active with session: 23da663b-503f-46aa-8af7-ad49c11b1847","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.444781017Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.444796178Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387,"session_id":"23da663b-503f-46aa-8af7-ad49c11b1847"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:51:58.444808868Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 27 len 8192 data: 5 5 03/30 Read block 562 len 40960 04/30 Read block 107 len 16384 05/30 Read block 109 len 8192 06/30 Flush 07/30 Write block 197 len 8192 data: 7 7 08/30 Read block 19 len 32768 09/30 Write block 387 len 32768 data: 5 5 5 3 3 3 4 4 10/30 Flush 11/30 Write block 149 len 40960 data: 6 5 5 6 3 4 5 5 5 5 12/30 Write block 147 len 32768 data: 6 6 7 6 6 7 4 5 13/30 Write block 227 len 28672 data: 2 2 3 3 3 3 5 14/30 Read block 209 len 24576 15/30 Write block 47 len 32768 data: 5 5 7 7 6 6 6 8 16/30 Write block 328 len 16384 data: 6 6 7 7 17/30 Write block 74 len 16384 data: 8 7 4 4 18/30 Write block 183 len 16384 data: 6 7 7 9 19/30 Write block 296 len 20480 data: 3 2 3 3 4 20/30 Write block 2 len 32768 data: 4 4 4 5 5 4 4 3 21/30 Write block 44 len 20480 data: 5 6 7 6 6 22/30 Read block 377 len 4096 23/30 Read block 354 len 36864 24/30 Read block 37 len 28672 25/30 Flush 26/30 Write block 507 len 40960 data: 7 6 6 5 5 7 7 7 6 5 27/30 Write block 244 len 12288 data: 5 6 7 28/30 Read block 555 len 40960 29/30 Write block 583 len 16384 data: 6 6 6 6 30/30 Write block 264 len 20480 data: 3 3 3 3 4 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 10 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:51:58.463069274Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1387} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 94 94 94 F F F 1 020-039 50 50 50 92 92 92 F F F 2 040-059 50 50 50 94 94 94 F F F 3 060-079 50 50 50 94 94 94 F F F 4 080-099 43 43 43 83 83 83 F F F 5 100-119 47 47 47 90 90 90 F F F 6 120-139 47 47 47 90 90 90 F F F 7 140-159 50 50 50 94 94 94 F F F 8 160-179 47 47 47 90 90 90 F F F 9 180-199 50 50 50 94 94 94 F F F 10 200-219 47 47 47 90 90 90 F F F 11 220-239 50 50 50 94 94 94 F F F 12 240-259 46 46 46 89 89 89 T T T 13 260-279 46 46 46 89 89 89 T T T 14 280-299 50 50 50 94 94 94 F F F 15 300-319 50 50 50 94 94 94 F F F 16 320-339 50 50 50 94 94 94 F F F 17 340-359 46 46 46 89 89 89 F F F 18 360-379 42 42 42 79 79 79 F F F 19 380-399 50 50 50 93 93 93 F F F 20 400-419 47 47 47 90 90 90 F F F 21 420-439 34 34 34 65 65 65 F F F 22 440-459 40 40 40 74 74 74 F F F 23 460-479 45 45 45 88 88 88 F F F 24 480-499 46 46 46 89 89 89 F F F 25 500-519 47 47 47 90 90 90 T T T 26 520-539 46 46 46 89 89 89 F F F 27 540-559 44 44 44 87 87 87 F F F 28 560-579 46 46 46 89 89 89 F F F 29 580-599 47 47 47 90 90 90 T T T Max gen: 50, Max flush: 94 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 94 94 94 F F F 1 020-039 50 50 50 92 92 92 F F F 2 040-059 50 50 50 94 94 94 F F F 3 060-079 50 50 50 94 94 94 F F F 4 080-099 43 43 43 83 83 83 F F F 5 100-119 47 47 47 90 90 90 F F F 6 120-139 47 47 47 90 90 90 F F F 7 140-159 50 50 50 94 94 94 F F F 8 160-179 47 47 47 90 90 90 F F F 9 180-199 50 50 50 94 94 94 F F F 10 200-219 47 47 47 90 90 90 F F F 11 220-239 50 50 50 94 94 94 F F F 12 240-259 51 51 51 95 95 95 F F F 13 260-279 51 51 51 95 95 95 F F F 14 280-299 50 50 50 94 94 94 F F F 15 300-319 50 50 50 94 94 94 F F F 16 320-339 50 50 50 94 94 94 F F F 17 340-359 46 46 46 89 89 89 F F F 18 360-379 42 42 42 79 79 79 F F F 19 380-399 50 50 50 93 93 93 F F F 20 400-419 47 47 47 90 90 90 F F F 21 420-439 34 34 34 65 65 65 F F F 22 440-459 40 40 40 74 74 74 F F F 23 460-479 45 45 45 88 88 88 F F F 24 480-499 46 46 46 89 89 89 F F F 25 500-519 51 51 51 95 95 95 F F F 26 520-539 46 46 46 89 89 89 F F F 27 540-559 44 44 44 87 87 87 F F F 28 560-579 46 46 46 89 89 89 F F F 29 580-599 51 51 51 95 95 95 F F F Max gen: 51, Max flush: 95 Begin loop 26 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.754374105Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:52:02.796884047Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.797241645Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.82878905Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.829334821Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.829371052Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.829385252Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.829399443Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395} {"msg":"Crucible e908d391-0226-431a-b9fd-e2f7d61708e4 has session id: cfa2c129-b536-4801-81dc-7e6177ff6676","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.829883913Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"Upstairs opts: Upstairs UUID: e908d391-0226-431a-b9fd-e2f7d61708e4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.829919074Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"Crucible stats registered with UUID: e908d391-0226-431a-b9fd-e2f7d61708e4","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.829935244Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.860662001Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.860707022Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395} {"msg":"e908d391-0226-431a-b9fd-e2f7d61708e4 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.860775973Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.860808834Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"client":"0","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.860839955Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"client":"0","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.860873995Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"client":"1","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.860902016Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"client":"1","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.860927036Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"client":"2","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.860953117Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"client":"2","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.860980798Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"":"io task","client":"2","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.861009378Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"":"io task","client":"0","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.861035689Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"":"io task","client":"1","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"ds_connection connected from Ok(127.0.0.1:63912)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.861062779Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"":"io task","client":"1","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"ds_connection connected from Ok(127.0.0.1:35952)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.861220083Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"":"io task","client":"2","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"ds_connection connected from Ok(127.0.0.1:38469)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.861279684Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"":"io task","client":"0","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.861776515Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"client":"1","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.861822936Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"client":"1","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.861856937Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"client":"2","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.861887477Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"client":"2","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.861916818Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"client":"0","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.861932258Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"client":"0","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.862399219Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"client":"2","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.862519751Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"client":"1","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.862551482Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"client":"0","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.862897299Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"client":"2","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.86293671Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"client":"2","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.862966371Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.863142445Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"client":"1","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.863171625Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"client":"1","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.863187496Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.863200606Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"client":"0","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.863214536Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"client":"0","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"[0]R flush_numbers[0..12]: [94, 92, 94, 94, 83, 90, 90, 94, 90, 94, 90, 94]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.863231546Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"":"downstairs","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"[0]R generation[0..12]: [50, 50, 50, 50, 43, 47, 47, 50, 47, 50, 47, 50]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.863245777Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"":"downstairs","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.863260127Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"":"downstairs","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"[1]R flush_numbers[0..12]: [94, 92, 94, 94, 83, 90, 90, 94, 90, 94, 90, 94]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.863274707Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"":"downstairs","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"[1]R generation[0..12]: [50, 50, 50, 50, 43, 47, 47, 50, 47, 50, 47, 50]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.863288388Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"":"downstairs","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.863302428Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"":"downstairs","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"[2]R flush_numbers[0..12]: [94, 92, 94, 94, 83, 90, 90, 94, 90, 94, 90, 94]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.863316079Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"":"downstairs","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"[2]R generation[0..12]: [50, 50, 50, 50, 43, 47, 47, 50, 47, 50, 47, 50]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.863329879Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"":"downstairs","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.863343989Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"":"downstairs","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.863357229Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"":"downstairs","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.86337055Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"":"downstairs","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"Next flush: 96","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:52:02.86338397Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"":"downstairs","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.86340172Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"":"downstairs","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.863415161Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.863428821Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"client":"0","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.863442631Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"client":"1","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.863455971Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"client":"2","":"downstairs","session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.863469232Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.863481722Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"e908d391-0226-431a-b9fd-e2f7d61708e4 is now active with session: cfa2c129-b536-4801-81dc-7e6177ff6676","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.863494832Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.863507123Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395,"session_id":"cfa2c129-b536-4801-81dc-7e6177ff6676"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:02.863519543Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Write block 227 len 40960 data: 3 3 4 4 4 4 6 6 6 6 04/30 Write block 88 len 36864 data: 4 6 5 5 5 5 6 8 6 05/30 Read block 579 len 4096 06/30 Read block 353 len 36864 07/30 Read block 431 len 12288 08/30 Write block 8 len 36864 data: 5 4 2 4 3 5 5 7 7 09/30 Read block 581 len 8192 10/30 Read block 197 len 32768 11/30 Read block 459 len 32768 12/30 Write block 539 len 20480 data: 6 4 2 2 2 13/30 Write block 12 len 8192 data: 4 6 14/30 Read block 395 len 20480 15/30 Read block 300 len 16384 16/30 Read block 312 len 20480 17/30 Read block 264 len 32768 18/30 Flush 19/30 Write block 323 len 4096 data: 4 20/30 Read block 498 len 24576 21/30 Write block 170 len 12288 data: 5 6 5 22/30 Write block 553 len 32768 data: 6 6 6 6 5 4 4 4 23/30 Flush 24/30 Write block 546 len 32768 data: 5 5 5 5 6 6 7 7 25/30 Flush 26/30 Write block 26 len 16384 data: 6 6 6 5 27/30 Write block 19 len 28672 data: 8 7 7 6 7 6 6 28/30 Read block 199 len 28672 29/30 Read block 549 len 28672 30/30 Read block 188 len 24576 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 4 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 6 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:52:02.884009397Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1395} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 98 98 98 T T T 1 020-039 50 50 50 92 92 92 T T T 2 040-059 50 50 50 94 94 94 F F F 3 060-079 50 50 50 94 94 94 F F F 4 080-099 52 52 52 98 98 98 F F F 5 100-119 47 47 47 90 90 90 F F F 6 120-139 47 47 47 90 90 90 F F F 7 140-159 50 50 50 94 94 94 F F F 8 160-179 52 52 52 99 99 99 F F F 9 180-199 50 50 50 94 94 94 F F F 10 200-219 47 47 47 90 90 90 F F F 11 220-239 52 52 52 98 98 98 F F F 12 240-259 51 51 51 95 95 95 F F F 13 260-279 51 51 51 95 95 95 F F F 14 280-299 50 50 50 94 94 94 F F F 15 300-319 50 50 50 94 94 94 F F F 16 320-339 52 52 52 99 99 99 F F F 17 340-359 46 46 46 89 89 89 F F F 18 360-379 42 42 42 79 79 79 F F F 19 380-399 50 50 50 93 93 93 F F F 20 400-419 47 47 47 90 90 90 F F F 21 420-439 34 34 34 65 65 65 F F F 22 440-459 40 40 40 74 74 74 F F F 23 460-479 45 45 45 88 88 88 F F F 24 480-499 46 46 46 89 89 89 F F F 25 500-519 51 51 51 95 95 95 F F F 26 520-539 52 52 52 98 98 98 F F F 27 540-559 52 52 52 100 100 100 F F F 28 560-579 52 52 52 99 99 99 F F F 29 580-599 51 51 51 95 95 95 F F F Max gen: 52, Max flush: 100 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 53 53 53 101 101 101 F F F 1 020-039 53 53 53 101 101 101 F F F 2 040-059 50 50 50 94 94 94 F F F 3 060-079 50 50 50 94 94 94 F F F 4 080-099 52 52 52 98 98 98 F F F 5 100-119 47 47 47 90 90 90 F F F 6 120-139 47 47 47 90 90 90 F F F 7 140-159 50 50 50 94 94 94 F F F 8 160-179 52 52 52 99 99 99 F F F 9 180-199 50 50 50 94 94 94 F F F 10 200-219 47 47 47 90 90 90 F F F 11 220-239 52 52 52 98 98 98 F F F 12 240-259 51 51 51 95 95 95 F F F 13 260-279 51 51 51 95 95 95 F F F 14 280-299 50 50 50 94 94 94 F F F 15 300-319 50 50 50 94 94 94 F F F 16 320-339 52 52 52 99 99 99 F F F 17 340-359 46 46 46 89 89 89 F F F 18 360-379 42 42 42 79 79 79 F F F 19 380-399 50 50 50 93 93 93 F F F 20 400-419 47 47 47 90 90 90 F F F 21 420-439 34 34 34 65 65 65 F F F 22 440-459 40 40 40 74 74 74 F F F 23 460-479 45 45 45 88 88 88 F F F 24 480-499 46 46 46 89 89 89 F F F 25 500-519 51 51 51 95 95 95 F F F 26 520-539 52 52 52 98 98 98 F F F 27 540-559 52 52 52 100 100 100 F F F 28 560-579 52 52 52 99 99 99 F F F 29 580-599 51 51 51 95 95 95 F F F Max gen: 53, Max flush: 101 Begin loop 27 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.163068133Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.206631908Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.2071981Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.207234221Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.207249821Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.207264282Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403} {"msg":"Crucible dc1bb8f6-0af1-449a-81f5-05d2dbd50901 has session id: f1214a69-981f-4680-89aa-9768c2c2601e","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.207689041Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"Upstairs opts: Upstairs UUID: dc1bb8f6-0af1-449a-81f5-05d2dbd50901, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.207722522Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"Crucible stats registered with UUID: dc1bb8f6-0af1-449a-81f5-05d2dbd50901","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.207738742Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.238416957Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.238475249Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403} {"msg":"dc1bb8f6-0af1-449a-81f5-05d2dbd50901 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.238490689Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.238504359Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"client":"0","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.23852802Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"client":"0","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.23854252Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"client":"1","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.238561291Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"client":"1","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.238621132Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"client":"2","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.238648933Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"client":"2","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.238676203Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"":"io task","client":"2","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.238702614Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"":"io task","client":"0","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.238728774Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"":"io task","client":"1","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"ds_connection connected from Ok(127.0.0.1:59520)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.238920908Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"":"io task","client":"0","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"ds_connection connected from Ok(127.0.0.1:49707)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.238964109Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"":"io task","client":"2","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"ds_connection connected from Ok(127.0.0.1:60428)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.23899427Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"":"io task","client":"1","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.239545222Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"client":"2","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.239584743Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"client":"2","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.239771677Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"client":"1","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.239816648Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"client":"1","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.239881459Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"client":"0","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.2399088Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"client":"0","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.240229187Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"client":"2","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.240295538Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"client":"0","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.24038554Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"client":"1","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.240799069Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"client":"2","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.24083882Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"client":"2","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.240867991Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.240894921Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"client":"0","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.240922402Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"client":"0","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.240945642Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.240968483Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"client":"1","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.240992823Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"client":"1","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"[0]R flush_numbers[0..12]: [101, 101, 94, 94, 98, 90, 90, 94, 99, 94, 90, 98]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.241019834Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"":"downstairs","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"[0]R generation[0..12]: [53, 53, 50, 50, 52, 47, 47, 50, 52, 50, 47, 52]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.241044765Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"":"downstairs","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.241069775Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"":"downstairs","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"[1]R flush_numbers[0..12]: [101, 101, 94, 94, 98, 90, 90, 94, 99, 94, 90, 98]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.241095915Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"":"downstairs","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"[1]R generation[0..12]: [53, 53, 50, 50, 52, 47, 47, 50, 52, 50, 47, 52]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.241127786Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"":"downstairs","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.241154977Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"":"downstairs","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"[2]R flush_numbers[0..12]: [101, 101, 94, 94, 98, 90, 90, 94, 99, 94, 90, 98]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.241183888Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"":"downstairs","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R generation[0..12]: [53, 53, 50, 50, 52, 47, 47, 50, 52, 50, 47, 52]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.241221958Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"":"downstairs","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.241254929Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"":"downstairs","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.24128491Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"":"downstairs","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.24131403Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"":"downstairs","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"Next flush: 102","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.241342701Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"":"downstairs","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.241372592Z","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"":"downstairs","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.241405312Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.241428193Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"client":"0","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.241442713Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"client":"1","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.241457183Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"client":"2","":"downstairs","session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.241470504Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.241483264Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"dc1bb8f6-0af1-449a-81f5-05d2dbd50901 is now active with session: f1214a69-981f-4680-89aa-9768c2c2601e","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.241499154Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.241512824Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403,"session_id":"f1214a69-981f-4680-89aa-9768c2c2601e"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:07.241525345Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 125 len 36864 data: 4 4 4 5 3 3 3 3 3 02/30 Flush 03/30 Write block 392 len 28672 data: 4 5 5 6 6 5 6 04/30 Read block 237 len 24576 05/30 Write block 515 len 16384 data: 7 6 6 6 06/30 Read block 260 len 36864 07/30 Read block 216 len 28672 08/30 Write block 131 len 20480 data: 4 4 4 5 5 09/30 Flush 10/30 Write block 62 len 16384 data: 6 5 6 6 11/30 Read block 295 len 12288 12/30 Read block 583 len 20480 13/30 Read block 556 len 40960 14/30 Write block 266 len 16384 data: 4 4 5 7 15/30 Read block 19 len 16384 16/30 Read block 363 len 36864 17/30 Write block 215 len 40960 data: 6 6 2 3 4 4 5 5 4 3 18/30 Read block 149 len 32768 19/30 Write block 297 len 8192 data: 3 4 20/30 Write block 110 len 32768 data: 3 4 5 5 6 7 6 5 21/30 Write block 87 len 28672 data: 4 5 7 6 6 6 6 22/30 Read block 466 len 12288 23/30 Flush 24/30 Write block 523 len 28672 data: 3 5 4 7 5 4 3 25/30 Read block 172 len 12288 26/30 Flush 27/30 Write block 461 len 16384 data: 4 4 5 5 28/30 Read block 13 len 12288 29/30 Read block 199 len 32768 30/30 Read block 492 len 8192 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 4 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 2 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 3 3 4 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:52:07.26203244Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1403} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 53 53 53 101 101 101 F F F 1 020-039 53 53 53 101 101 101 F F F 2 040-059 50 50 50 94 94 94 F F F 3 060-079 54 54 54 104 104 104 F F F 4 080-099 54 54 54 104 104 104 F F F 5 100-119 54 54 54 104 104 104 F F F 6 120-139 54 54 54 103 103 103 F F F 7 140-159 50 50 50 94 94 94 F F F 8 160-179 52 52 52 99 99 99 F F F 9 180-199 50 50 50 94 94 94 F F F 10 200-219 54 54 54 104 104 104 F F F 11 220-239 54 54 54 104 104 104 F F F 12 240-259 51 51 51 95 95 95 F F F 13 260-279 54 54 54 104 104 104 F F F 14 280-299 54 54 54 104 104 104 F F F 15 300-319 50 50 50 94 94 94 F F F 16 320-339 52 52 52 99 99 99 F F F 17 340-359 46 46 46 89 89 89 F F F 18 360-379 42 42 42 79 79 79 F F F 19 380-399 54 54 54 103 103 103 F F F 20 400-419 47 47 47 90 90 90 F F F 21 420-439 34 34 34 65 65 65 F F F 22 440-459 40 40 40 74 74 74 F F F 23 460-479 45 45 45 88 88 88 T T T 24 480-499 46 46 46 89 89 89 F F F 25 500-519 54 54 54 103 103 103 F F F 26 520-539 54 54 54 105 105 105 F F F 27 540-559 52 52 52 100 100 100 F F F 28 560-579 52 52 52 99 99 99 F F F 29 580-599 51 51 51 95 95 95 F F F Max gen: 54, Max flush: 105 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 53 53 53 101 101 101 F F F 1 020-039 53 53 53 101 101 101 F F F 2 040-059 50 50 50 94 94 94 F F F 3 060-079 54 54 54 104 104 104 F F F 4 080-099 54 54 54 104 104 104 F F F 5 100-119 54 54 54 104 104 104 F F F 6 120-139 54 54 54 103 103 103 F F F 7 140-159 50 50 50 94 94 94 F F F 8 160-179 52 52 52 99 99 99 F F F 9 180-199 50 50 50 94 94 94 F F F 10 200-219 54 54 54 104 104 104 F F F 11 220-239 54 54 54 104 104 104 F F F 12 240-259 51 51 51 95 95 95 F F F 13 260-279 54 54 54 104 104 104 F F F 14 280-299 54 54 54 104 104 104 F F F 15 300-319 50 50 50 94 94 94 F F F 16 320-339 52 52 52 99 99 99 F F F 17 340-359 46 46 46 89 89 89 F F F 18 360-379 42 42 42 79 79 79 F F F 19 380-399 54 54 54 103 103 103 F F F 20 400-419 47 47 47 90 90 90 F F F 21 420-439 34 34 34 65 65 65 F F F 22 440-459 40 40 40 74 74 74 F F F 23 460-479 55 55 55 106 106 106 F F F 24 480-499 46 46 46 89 89 89 F F F 25 500-519 54 54 54 103 103 103 F F F 26 520-539 54 54 54 105 105 105 F F F 27 540-559 52 52 52 100 100 100 F F F 28 560-579 52 52 52 99 99 99 F F F 29 580-599 51 51 51 95 95 95 F F F Max gen: 55, Max flush: 106 Begin loop 28 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.508757614Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:52:11.551679665Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.552062074Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.583807822Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.584364174Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.584397315Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.584415215Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.584430306Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411} {"msg":"Crucible d5e502d5-c6ff-4cc7-82fe-8bca293d7355 has session id: 966da397-13eb-45b0-83ca-9fb2f6b39a69","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.584903416Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"Upstairs opts: Upstairs UUID: d5e502d5-c6ff-4cc7-82fe-8bca293d7355, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.584950637Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"Crucible stats registered with UUID: d5e502d5-c6ff-4cc7-82fe-8bca293d7355","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.584977208Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.615658563Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.615690294Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411} {"msg":"d5e502d5-c6ff-4cc7-82fe-8bca293d7355 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.615713694Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.615730145Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"client":"0","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.615758975Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"client":"0","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.615777826Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"client":"1","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.615795586Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"client":"1","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.615813657Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"client":"2","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.615848507Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"client":"2","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.615880598Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"":"io task","client":"2","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.615912389Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"":"io task","client":"0","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.615945639Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"":"io task","client":"1","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"ds_connection connected from Ok(127.0.0.1:61577)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.61597755Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"":"io task","client":"2","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"ds_connection connected from Ok(127.0.0.1:44277)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.616010971Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"":"io task","client":"0","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"ds_connection connected from Ok(127.0.0.1:62536)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.616045912Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"":"io task","client":"1","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.616695686Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"client":"1","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.616726236Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"client":"1","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.616742207Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"client":"2","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.616757467Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"client":"2","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.616828779Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"client":"0","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.616847429Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"client":"0","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.61736315Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"client":"1","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.617390831Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"client":"2","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.617406231Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"client":"0","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.617940473Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"client":"1","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.617977123Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"client":"1","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.618007504Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.618024685Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"client":"2","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.618038965Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"client":"2","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.618052695Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.618065065Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"client":"0","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.618078416Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"client":"0","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"[0]R flush_numbers[0..12]: [101, 101, 94, 104, 104, 104, 103, 94, 99, 94, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.618092316Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"":"downstairs","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"[0]R generation[0..12]: [53, 53, 50, 54, 54, 54, 54, 50, 52, 50, 54, 54]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.618106156Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"":"downstairs","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.618120197Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"":"downstairs","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"[1]R flush_numbers[0..12]: [101, 101, 94, 104, 104, 104, 103, 94, 99, 94, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.618134017Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"":"downstairs","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"[1]R generation[0..12]: [53, 53, 50, 54, 54, 54, 54, 50, 52, 50, 54, 54]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.618147747Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"":"downstairs","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.618161738Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"":"downstairs","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"[2]R flush_numbers[0..12]: [101, 101, 94, 104, 104, 104, 103, 94, 99, 94, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.618179638Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"":"downstairs","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"[2]R generation[0..12]: [53, 53, 50, 54, 54, 54, 54, 50, 52, 50, 54, 54]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.618194338Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"":"downstairs","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":Wait for a query_work_queue command to finish before sending IO "[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.618221339Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"":"downstairs","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.61824864Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"":"downstairs","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.61827345Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"":"downstairs","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"Next flush: 107","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.618295681Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"":"downstairs","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.618320191Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"":"downstairs","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible",Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "level":30,"time":"2024-11-28T04:52:11.618347702Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.618371092Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"client":"0","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.618394953Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"client":"1","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.618410033Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"client":"2","":"downstairs","session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.618424573Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.618437273Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"d5e502d5-c6ff-4cc7-82fe-8bca293d7355 is now active with session: 966da397-13eb-45b0-83ca-9fb2f6b39a69","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.618450204Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.618462364Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411,"session_id":"966da397-13eb-45b0-83ca-9fb2f6b39a69"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:11.618476894Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 574 len 24576 data: 6 7 6 6 6 6 02/30 Write block 569 len 28672 data: 6 6 5 5 6 7 8 03/30 Read block 143 len 8192 04/30 Flush 05/30 Flush 06/30 Write block 38 len 24576 data: 3 3 3 3 4 4 07/30 Read block 308 len 40960 08/30 Write block 545 len 4096 data: 4 09/30 Read block 260 len 12288 10/30 Write block 21 len 24576 data: 8 7 8 7 7 7 11/30 Write block 517 len 12288 data: 7 7 7 12/30 Write block 260 len 32768 data: 4 4 3 2 4 4 5 5 13/30 Write block 73 len 28672 data: 9 9 8 5 5 4 4 14/30 Write block 459 len 16384 data: 4 4 5 5 15/30 Read block 166 len 4096 16/30 Read block 430 len 36864 17/30 Flush 18/30 Write block 453 len 32768 data: 5 6 4 5 4 3 5 5 19/30 Write block 36 len 28672 data: 3 3 4 4 4 4 5 20/30 Read block 546 len 32768 21/30 Read block 522 len 20480 22/30 Write block 529 len 40960 data: 4 2 3 3 4 5 4 4 6 7 23/30 Read block 402 len 40960 24/30 Read block 551 len 32768 25/30 Read block 100 len 40960 26/30 Read block 489 len 28672 27/30 Write block 399 len 12288 data: 8 7 7 28/30 Write block 338 len 40960 data: 8 7 7 5 6 6 7 7 6 5 29/30 Write block 189 len 24576 data: 8 7 7 6 7 8 30/30 Read block 402 len 20480 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 8 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 6 Done Sent Done false 1029 Acked Read 5 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 0 3 Done 12 11 13 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:52:11.637744852Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1411} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 53 53 53 101 101 101 F F F 1 020-039 56 56 56 109 109 109 T T T 2 040-059 56 56 56 109 109 109 T T T 3 060-079 56 56 56 109 109 109 F F F 4 080-099 54 54 54 104 104 104 F F F 5 100-119 54 54 54 104 104 104 F F F 6 120-139 54 54 54 103 103 103 F F F 7 140-159 50 50 50 94 94 94 F F F 8 160-179 52 52 52 99 99 99 F F F 9 180-199 50 50 50 94 94 94 T T T 10 200-219 54 54 54 104 104 104 F F F 11 220-239 54 54 54 104 104 104 F F F 12 240-259 51 51 51 95 95 95 F F F 13 260-279 56 56 56 109 109 109 F F F 14 280-299 54 54 54 104 104 104 F F F 15 300-319 50 50 50 94 94 94 F F F 16 320-339 52 52 52 99 99 99 T T T 17 340-359 46 46 46 89 89 89 T T T 18 360-379 42 42 42 79 79 79 F F F 19 380-399 54 54 54 103 103 103 T T T 20 400-419 47 47 47 90 90 90 T T T 21 420-439 34 34 34 65 65 65 F F F 22 440-459 56 56 56 109 109 109 T T T 23 460-479 56 56 56 109 109 109 T T T 24 480-499 46 46 46 89 89 89 F F F 25 500-519 56 56 56 109 109 109 F F F 26 520-539 54 54 54 105 105 105 T T T 27 540-559 56 56 56 109 109 109 F F F 28 560-579 56 56 56 107 107 107 F F F 29 580-599 51 51 51 95 95 95 F F F Max gen: 56, Max flush: 109 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 53 53 53 101 101 101 F F F 1 020-039 57 57 57 110 110 110 F F F 2 040-059 57 57 57 110 110 110 F F F 3 060-079 56 56 56 109 109 109 F F F 4 080-099 54 54 54 104 104 104 F F F 5 100-119 54 54 54 104 104 104 F F F 6 120-139 54 54 54 103 103 103 F F F 7 140-159 50 50 50 94 94 94 F F F 8 160-179 52 52 52 99 99 99 F F F 9 180-199 57 57 57 110 110 110 F F F 10 200-219 54 54 54 104 104 104 F F F 11 220-239 54 54 54 104 104 104 F F F 12 240-259 51 51 51 95 95 95 F F F 13 260-279 56 56 56 109 109 109 F F F 14 280-299 54 54 54 104 104 104 F F F 15 300-319 50 50 50 94 94 94 F F F 16 320-339 57 57 57 110 110 110 F F F 17 340-359 57 57 57 110 110 110 F F F 18 360-379 42 42 42 79 79 79 F F F 19 380-399 57 57 57 110 110 110 F F F 20 400-419 57 57 57 110 110 110 F F F 21 420-439 34 34 34 65 65 65 F F F 22 440-459 57 57 57 110 110 110 F F F 23 460-479 57 57 57 110 110 110 F F F 24 480-499 46 46 46 89 89 89 F F F 25 500-519 56 56 56 109 109 109 F F F 26 520-539 57 57 57 110 110 110 F F F 27 540-559 56 56 56 109 109 109 F F F 28 560-579 56 56 56 107 107 107 F F F 29 580-599 51 51 51 95 95 95 F F F Max gen: 57, Max flush: 110 Begin loop 29 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:15.958779788Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.002456346Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.003195802Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.003235012Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.003256963Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.003274523Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419} {"msg":"Crucible 5518457e-64bb-47b3-aa15-62c7310bbaa9 has session id: 77829aa9-67d6-4d80-a4f1-cd1a4642dae9","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.003699593Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"Upstairs opts: Upstairs UUID: 5518457e-64bb-47b3-aa15-62c7310bbaa9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.003747104Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"Crucible stats registered with UUID: 5518457e-64bb-47b3-aa15-62c7310bbaa9","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.003772884Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.03492155Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.034951651Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419} {"msg":"5518457e-64bb-47b3-aa15-62c7310bbaa9 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.034996332Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.035022242Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"client":"0","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.035047073Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"client":"0","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.035061283Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"client":"1","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.035074903Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"client":"1","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.035088394Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"client":"2","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.035102034Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"client":"2","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.035115474Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"":"io task","client":"2","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.035142255Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"":"io task","client":"0","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.035171065Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"":"io task","client":"1","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"ds_connection connected from Ok(127.0.0.1:41037)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.035199336Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"":"io task","client":"2","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"ds_connection connected from Ok(127.0.0.1:40625)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.035234607Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"":"io task","client":"0","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"ds_connection connected from Ok(127.0.0.1:37677)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.035263797Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"":"io task","client":"1","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.036005403Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"client":"1","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.036034774Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"client":"1","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.036050404Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"client":"0","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.036064685Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"client":"0","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.036088775Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"client":"2","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.036104456Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"client":"2","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.036610707Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"client":"1","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.036642267Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"client":"0","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.036672398Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"client":"2","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.037192029Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"client":"1","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.03722365Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"client":"1","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.03723991Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.037308652Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"client":"0","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.037329922Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"client":"0","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.037343743Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.037356343Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"client":"2","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.037369653Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"client":"2","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"[0]R flush_numbers[0..12]: [101, 110, 110, 109, 104, 104, 103, 94, 99, 110, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.037383673Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"":"downstairs","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"[0]R generation[0..12]: [53, 57, 57, 56, 54, 54, 54, 50, 52, 57, 54, 54]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.037398334Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"":"downstairs","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.037412534Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"":"downstairs","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"[1]R flush_numbers[0..12]: [101, 110, 110, 109, 104, 104, 103, 94, 99, 110, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.037426494Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"":"downstairs","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"[1]R generation[0..12]: [53, 57, 57, 56, 54, 54, 54, 50, 52, 57, 54, 54]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.037440365Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"":"downstairs","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.037454425Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"":"downstairs","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"[2]R flush_numbers[0..12]: [101, 110, 110, 109, 104, 104, 103, 94, 99, 110, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.037468215Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"":"downstairs","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"[2]R generation[0..12]: [53, 57, 57, 56, 54, 54, 54, 50, 52, 57, 54, 54]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.037481875Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"":"downstairs","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.037499076Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"":"downstairs","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.037512556Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"":"downstairs","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.037525916Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"":"downstairs","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {Wait for a query_work_queue command to finish before sending IO "msg":"Next flush: 111","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.037563147Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"":"downstairs","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.037576748Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"":"downstairs","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.037590048Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.037603168Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"client":"0","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.037617088Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"client":"1","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.037630609Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"client":"2","":"downstairs","session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.037643989Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.037656489Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"5518457e-64bb-47b3-aa15-62c7310bbaa9 is now active with session: 77829aa9-67d6-4d80-a4f1-cd1a4642dae9","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.037669499Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.03768158Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419,"session_id":"77829aa9-67d6-4d80-a4f1-cd1a4642dae9"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:16.03769545Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 474 len 20480 data: 7 7 7 6 6 02/30 Write block 227 len 4096 data: 4 03/30 Flush 04/30 Flush 05/30 Write block 495 len 24576 data: 5 4 4 4 4 7 06/30 Write block 478 len 40960 data: 7 5 6 6 6 4 3 3 4 4 07/30 Read block 430 len 16384 08/30 Read block 323 len 12288 09/30 Read block 207 len 36864 10/30 Read block 187 len 32768 11/30 Read block 541 len 40960 12/30 Write block 334 len 8192 data: 8 9 13/30 Write block 515 len 16384 data: 8 7 8 8 14/30 Read block 454 len 28672 15/30 Write block 347 len 4096 data: 6 16/30 Read block 253 len 24576 17/30 Write block 410 len 4096 data: 4 18/30 Read block 545 len 4096 19/30 Read block 468 len 32768 20/30 Read block 111 len 24576 21/30 Read block 446 len 36864 22/30 Write block 556 len 20480 data: 7 6 5 5 5 23/30 Write block 466 len 16384 data: 3 4 5 5 24/30 Read block 394 len 16384 25/30 Read block 297 len 16384 26/30 Read block 313 len 16384 27/30 Write block 341 len 4096 data: 6 28/30 Read block 189 len 16384 29/30 Read block 313 len 32768 30/30 Write block 554 len 40960 data: 7 7 8 7 6 6 6 3 3 3 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 6 Done Done Done false 1005 Acked Write 10 Done Done Done false 1006 Acked Read 4 Done Done Done false 1007 Acked Read 3 Done Done Done false 1008 Acked Read 9 Done Done Done false 1009 Acked Read 8 Done Done Done false 1010 Acked Read 10 Done Done Done false 1011 Acked Write 2 Done Done Done false 1012 Acked Write 4 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 25 25 25 75 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:52:16.056443067Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1419} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 53 53 53 101 101 101 F F F 1 020-039 57 57 57 110 110 110 F F F 2 040-059 57 57 57 110 110 110 F F F 3 060-079 56 56 56 109 109 109 F F F 4 080-099 54 54 54 104 104 104 F F F 5 100-119 54 54 54 104 104 104 F F F 6 120-139 54 54 54 103 103 103 F F F 7 140-159 50 50 50 94 94 94 F F F 8 160-179 52 52 52 99 99 99 F F F 9 180-199 57 57 57 110 110 110 F F F 10 200-219 54 54 54 104 104 104 F F F 11 220-239 58 58 58 111 111 111 F F F 12 240-259 51 51 51 95 95 95 F F F 13 260-279 56 56 56 109 109 109 F F F 14 280-299 54 54 54 104 104 104 F F F 15 300-319 50 50 50 94 94 94 F F F 16 320-339 57 57 57 110 110 110 T T T 17 340-359 57 57 57 110 110 110 T T T 18 360-379 42 42 42 79 79 79 F F F 19 380-399 57 57 57 110 110 110 F F F 20 400-419 57 57 57 110 110 110 T T T 21 420-439 34 34 34 65 65 65 F F F 22 440-459 57 57 57 110 110 110 F F F 23 460-479 58 58 58 111 111 111 T T T 24 480-499 46 46 46 89 89 89 T T T 25 500-519 56 56 56 109 109 109 T T T 26 520-539 57 57 57 110 110 110 F F F 27 540-559 56 56 56 109 109 109 T T T 28 560-579 56 56 56 107 107 107 T T T 29 580-599 51 51 51 95 95 95 F F F Max gen: 58, Max flush: 111 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 53 53 53 101 101 101 F F F 1 020-039 57 57 57 110 110 110 F F F 2 040-059 57 57 57 110 110 110 F F F 3 060-079 56 56 56 109 109 109 F F F 4 080-099 54 54 54 104 104 104 F F F 5 100-119 54 54 54 104 104 104 F F F 6 120-139 54 54 54 103 103 103 F F F 7 140-159 50 50 50 94 94 94 F F F 8 160-179 52 52 52 99 99 99 F F F 9 180-199 57 57 57 110 110 110 F F F 10 200-219 54 54 54 104 104 104 F F F 11 220-239 58 58 58 111 111 111 F F F 12 240-259 51 51 51 95 95 95 F F F 13 260-279 56 56 56 109 109 109 F F F 14 280-299 54 54 54 104 104 104 F F F 15 300-319 50 50 50 94 94 94 F F F 16 320-339 59 59 59 112 112 112 F F F 17 340-359 59 59 59 112 112 112 F F F 18 360-379 42 42 42 79 79 79 F F F 19 380-399 57 57 57 110 110 110 F F F 20 400-419 59 59 59 112 112 112 F F F 21 420-439 34 34 34 65 65 65 F F F 22 440-459 57 57 57 110 110 110 F F F 23 460-479 59 59 59 112 112 112 F F F 24 480-499 59 59 59 112 112 112 F F F 25 500-519 59 59 59 112 112 112 F F F 26 520-539 57 57 57 110 110 110 F F F 27 540-559 59 59 59 112 112 112 F F F 28 560-579 59 59 59 112 112 112 F F F 29 580-599 51 51 51 95 95 95 F F F Max gen: 59, Max flush: 112 Begin loop 30 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.343501636Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.387613493Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.388297958Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.388352249Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.38838132Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.38839905Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427} {"msg":"Crucible 3fa7c22d-3e04-4139-bd07-acdec68f96c6 has session id: 522c3f8a-5a2f-4441-920b-f12d356d5475","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.3888274Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"Upstairs opts: Upstairs UUID: 3fa7c22d-3e04-4139-bd07-acdec68f96c6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.38887371Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"Crucible stats registered with UUID: 3fa7c22d-3e04-4139-bd07-acdec68f96c6","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.388891331Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.419561686Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.419596507Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427} {"msg":"3fa7c22d-3e04-4139-bd07-acdec68f96c6 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.419665709Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.419701379Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"client":"0","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.41973438Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"client":"0","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.419758711Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"client":"1","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.419792871Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"client":"1","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.419816452Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"client":"2","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.419847813Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"client":"2","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.419878813Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"":"io task","client":"2","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.419919164Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"":"io task","client":"0","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.419956735Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"":"io task","client":"1","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"ds_connection connected from Ok(127.0.0.1:65244)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.419988785Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"":"io task","client":"2","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"ds_connection connected from Ok(127.0.0.1:43848)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.420019436Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"":"io task","client":"0","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"ds_connection connected from Ok(127.0.0.1:49911)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.420126959Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"":"io task","client":"1","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.420764422Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"client":"0","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.420792943Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"client":"0","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.420807933Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"client":"1","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.420821754Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"client":"1","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.421006298Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"client":"2","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.421043228Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"client":"2","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.421521949Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"client":"0","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.42156718Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"client":"2","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.42159359Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"client":"1","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.422099971Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"client":"1","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.422145972Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"client":"1","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.422178403Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.422202044Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"client":"0","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.422225854Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"client":"0","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.422250025Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.422272205Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"client":"2","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.422297595Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"client":"2","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"[0]R flush_numbers[0..12]: [101, 110, 110, 109, 104, 104, 103, 94, 99, 110, 104, 111]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.422322116Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"":"downstairs","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"[0]R generation[0..12]: [53, 57, 57, 56, 54, 54, 54, 50, 52, 57, 54, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.422345707Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"":"downstairs","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.422372047Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"":"downstairs","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"[1]R flush_numbers[0..12]: [101, 110, 110, 109, 104, 104, 103, 94, 99, 110, 104, 111]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.422400008Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"":"downstairs","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"[1]R generation[0..12]: [53, 57, 57, 56, 54, 54, 54, 50, 52, 57, 54, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.422426058Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"":"downstairs","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.422452979Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"":"downstairs","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"[2]R flush_numbers[0..12]: [101, 110, 110, 109, 104, 104, 103, 94, 99, 110, 104, 111]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.42248268Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"":"downstairs","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R generation[0..12]: [53, 57, 57, 56, 54, 54, 54, 50, 52, 57, 54, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.422521871Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"":"downstairs","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.422557451Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"":"downstairs","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.422587542Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"":"downstairs","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.422617273Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"":"downstairs","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"Next flush: 113","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.422645733Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"":"downstairs","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.422675094Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"":"downstairs","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.422708075Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.422730095Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"client":"0","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.422744345Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"client":"1","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.422757796Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"client":"2","":"downstairs","session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.422771436Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.422783936Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"3fa7c22d-3e04-4139-bd07-acdec68f96c6 is now active with session: 522c3f8a-5a2f-4441-920b-f12d356d5475","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.422796926Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.422808937Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427,"session_id":"522c3f8a-5a2f-4441-920b-f12d356d5475"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:20.422821237Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 23 len 16384 data: 9 8 8 8 02/30 Read block 295 len 16384 03/30 Write block 270 len 12288 data: 10 9 9 04/30 Write block 39 len 40960 data: 5 5 5 6 5 6 7 8 7 7 05/30 Write block 566 len 36864 data: 3 4 4 7 7 6 6 7 8 06/30 Flush 07/30 Write block 494 len 32768 data: 5 6 5 5 5 5 8 6 08/30 Write block 364 len 32768 data: 3 3 3 3 4 5 5 6 09/30 Read block 284 len 32768 10/30 Read block 418 len 4096 11/30 Write block 382 len 32768 data: 4 3 3 3 3 6 6 6 12/30 Write block 77 len 4096 data: 6 13/30 Read block 261 len 36864 14/30 Read block 380 len 28672 15/30 Write block 521 len 28672 data: 6 4 4 6 5 8 6 16/30 Read block 554 len 20480 17/30 Read block 77 len 4096 18/30 Write block 255 len 32768 data: 6 6 7 6 5 5 5 4 19/30 Read block 370 len 12288 20/30 Read block 128 len 28672 21/30 Write block 500 len 20480 data: 9 7 5 7 5 22/30 Write block 455 len 24576 data: 5 6 5 4 6 6 23/30 Flush 24/30 Write block 457 len 12288 data: 6 5 7 25/30 Read block 178 len 36864 26/30 Write block 187 len 24576 data: 9 9 9 8 8 7 27/30 Write block 232 len 16384 data: 5 7 7 7 28/30 Read block 499 len 28672 29/30 Write block 410 len 24576 data: 5 3 3 4 4 4 30/30 Read block 532 len 36864 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 3 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:52:20.443175928Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1427} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 53 53 53 101 101 101 F F F 1 020-039 60 60 60 113 113 113 F F F 2 040-059 60 60 60 113 113 113 F F F 3 060-079 60 60 60 114 114 114 F F F 4 080-099 54 54 54 104 104 104 F F F 5 100-119 54 54 54 104 104 104 F F F 6 120-139 54 54 54 103 103 103 F F F 7 140-159 50 50 50 94 94 94 F F F 8 160-179 52 52 52 99 99 99 F F F 9 180-199 57 57 57 110 110 110 T T T 10 200-219 54 54 54 104 104 104 F F F 11 220-239 58 58 58 111 111 111 T T T 12 240-259 60 60 60 114 114 114 F F F 13 260-279 60 60 60 114 114 114 F F F 14 280-299 54 54 54 104 104 104 F F F 15 300-319 50 50 50 94 94 94 F F F 16 320-339 59 59 59 112 112 112 F F F 17 340-359 59 59 59 112 112 112 F F F 18 360-379 60 60 60 114 114 114 F F F 19 380-399 60 60 60 114 114 114 F F F 20 400-419 59 59 59 112 112 112 T T T 21 420-439 34 34 34 65 65 65 F F F 22 440-459 60 60 60 114 114 114 T T T 23 460-479 60 60 60 114 114 114 F F F 24 480-499 60 60 60 114 114 114 F F F 25 500-519 60 60 60 114 114 114 F F F 26 520-539 60 60 60 114 114 114 F F F 27 540-559 59 59 59 112 112 112 F F F 28 560-579 60 60 60 113 113 113 F F F 29 580-599 51 51 51 95 95 95 F F F Max gen: 60, Max flush: 114 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 53 53 53 101 101 101 F F F 1 020-039 60 60 60 113 113 113 F F F 2 040-059 60 60 60 113 113 113 F F F 3 060-079 60 60 60 114 114 114 F F F 4 080-099 54 54 54 104 104 104 F F F 5 100-119 54 54 54 104 104 104 F F F 6 120-139 54 54 54 103 103 103 F F F 7 140-159 50 50 50 94 94 94 F F F 8 160-179 52 52 52 99 99 99 F F F 9 180-199 61 61 61 115 115 115 F F F 10 200-219 54 54 54 104 104 104 F F F 11 220-239 61 61 61 115 115 115 F F F 12 240-259 60 60 60 114 114 114 F F F 13 260-279 60 60 60 114 114 114 F F F 14 280-299 54 54 54 104 104 104 F F F 15 300-319 50 50 50 94 94 94 F F F 16 320-339 59 59 59 112 112 112 F F F 17 340-359 59 59 59 112 112 112 F F F 18 360-379 60 60 60 114 114 114 F F F 19 380-399 60 60 60 114 114 114 F F F 20 400-419 61 61 61 115 115 115 F F F 21 420-439 34 34 34 65 65 65 F F F 22 440-459 61 61 61 115 115 115 F F F 23 460-479 60 60 60 114 114 114 F F F 24 480-499 60 60 60 114 114 114 F F F 25 500-519 60 60 60 114 114 114 F F F 26 520-539 60 60 60 114 114 114 F F F 27 540-559 59 59 59 112 112 112 F F F 28 560-579 60 60 60 113 113 113 F F F 29 580-599 51 51 51 95 95 95 F F F Max gen: 61, Max flush: 115 Begin loop 31 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.702990947Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:52:24.746598793Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.74695345Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.77968511Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.780293443Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.780328564Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.780343154Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.780358435Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435} {"msg":"Crucible 383f4643-9fa7-4159-930e-bfd1e54ff12b has session id: 8e3af720-3be7-49ee-ba9d-73eaeb1c971a","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.780814615Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"Upstairs opts: Upstairs UUID: 383f4643-9fa7-4159-930e-bfd1e54ff12b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.780861356Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"Crucible stats registered with UUID: 383f4643-9fa7-4159-930e-bfd1e54ff12b","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.780885726Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.811597612Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.811633243Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435} {"msg":"383f4643-9fa7-4159-930e-bfd1e54ff12b active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.811695705Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.811731585Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"client":"0","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.811764776Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"client":"0","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.811787027Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"client":"1","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.811859908Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"client":"1","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.811888709Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"client":"2","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.811914959Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"client":"2","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.81194179Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"":"io task","client":"2","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.811970471Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"":"io task","client":"0","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.812104514Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"":"io task","client":"1","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"ds_connection connected from Ok(127.0.0.1:48979)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.812158725Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"":"io task","client":"2","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"ds_connection connected from Ok(127.0.0.1:56105)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.812181535Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"":"io task","client":"1","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"ds_connection connected from Ok(127.0.0.1:59099)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.812200885Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"":"io task","client":"0","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.812787268Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"client":"1","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.812818899Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"client":"1","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.812835779Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"client":"2","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.81285043Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"client":"2","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.812906301Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"client":"0","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.812921131Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"client":"0","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.813480183Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"client":"1","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.813510134Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"client":"0","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.813567165Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"client":"2","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.814117617Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"client":"1","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.814146388Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"client":"1","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.814162368Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.814179959Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"client":"2","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.814196129Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"client":"2","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.814210549Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.81422438Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"client":"0","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.81423922Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"client":"0","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"[0]R flush_numbers[0..12]: [101, 113, 113, 114, 104, 104, 103, 94, 99, 115, 104, 115]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.81425426Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"":"downstairs","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"[0]R generation[0..12]: [53, 60, 60, 60, 54, 54, 54, 50, 52, 61, 54, 61]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.8142694Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"":"downstairs","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.814284901Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"":"downstairs","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"[1]R flush_numbers[0..12]: [101, 113, 113, 114, 104, 104, 103, 94, 99, 115, 104, 115]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.814300421Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"":"downstairs","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"[1]R generation[0..12]: [53, 60, 60, 60, 54, 54, 54, 50, 52, 61, 54, 61]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.814315561Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"":"downstairs","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.814331282Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"":"downstairs","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"[2]R flush_numbers[0..12]: [101, 113, 113, 114, 104, 104, 103, 94, 99, 115, 104, 115]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.814346592Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"":"downstairs","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"[2]R generation[0..12]: [53, 60, 60, 60, 54, 54, 54, 50, 52, 61, 54, 61]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.814361192Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"":"downstairs","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.814376273Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"":"downstairs","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.814399433Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"":"downstairs","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.814414124Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"":"downstairs","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"Next flush: 116","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.814428444Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"":"downstairs","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.814442684Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"":"downstairs","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.814457285Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.814470955Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"client":"0","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.814485325Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"client":"1","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.814499585Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"client":"2","":"downstairs","session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.814514166Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.814527996Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"383f4643-9fa7-4159-930e-bfd1e54ff12b is now active with session: 8e3af720-3be7-49ee-ba9d-73eaeb1c971a","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.814542226Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.814554887Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435,"session_id":"8e3af720-3be7-49ee-ba9d-73eaeb1c971a"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:24.814569257Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 101 len 16384 data: 5 6 6 4 03/30 Write block 461 len 12288 data: 6 6 6 04/30 Read block 530 len 36864 05/30 Write block 299 len 16384 data: 4 5 5 5 06/30 Write block 50 len 36864 data: 8 7 7 7 9 8 10 10 10 07/30 Write block 419 len 12288 data: 2 2 4 08/30 Write block 39 len 8192 data: 6 6 09/30 Write block 67 len 8192 data: 8 9 10/30 Write block 360 len 36864 data: 4 4 3 3 4 4 4 4 5 11/30 Flush 12/30 Write block 91 len 4096 data: 7 13/30 Flush 14/30 Write block 28 len 32768 data: 7 6 2 3 3 3 3 3 15/30 Write block 461 len 16384 data: 7 7 7 6 16/30 Read block 328 len 40960 17/30 Flush 18/30 Flush 19/30 Write block 73 len 8192 data: 10 10 20/30 Write block 104 len 12288 data: 5 4 3 21/30 Write block 78 len 8192 data: 5 5 22/30 Read block 66 len 20480 23/30 Read block 160 len 32768 24/30 Write block 361 len 12288 data: 5 4 4 25/30 Write block 36 len 24576 data: 4 4 5 7 7 6 26/30 Write block 18 len 16384 data: 6 9 8 9 27/30 Write block 390 len 12288 data: 4 4 5 28/30 Write block 498 len 36864 data: 6 6 10 8 6 8 6 5 6 29/30 Write block 47 len 8192 data: 8 8 30/30 Write block 303 len 4096 data: 5 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 2 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 9 Sent Sent Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:52:24.830074233Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1435} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 53 53 53 101 101 101 T T T 1 020-039 62 62 62 119 119 119 T T T 2 040-059 62 62 62 117 117 117 T T T 3 060-079 62 62 62 117 117 117 T T T 4 080-099 62 62 62 118 118 118 F F F 5 100-119 62 62 62 117 117 117 T T T 6 120-139 54 54 54 103 103 103 F F F 7 140-159 50 50 50 94 94 94 F F F 8 160-179 52 52 52 99 99 99 F F F 9 180-199 61 61 61 115 115 115 F F F 10 200-219 54 54 54 104 104 104 F F F 11 220-239 61 61 61 115 115 115 F F F 12 240-259 60 60 60 114 114 114 F F F 13 260-279 60 60 60 114 114 114 F F F 14 280-299 62 62 62 117 117 117 F F F 15 300-319 62 62 62 117 117 117 T T T 16 320-339 59 59 59 112 112 112 F F F 17 340-359 59 59 59 112 112 112 F F F 18 360-379 62 62 62 117 117 117 T T T 19 380-399 60 60 60 114 114 114 T T T 20 400-419 62 62 62 117 117 117 F F F 21 420-439 62 62 62 117 117 117 F F F 22 440-459 61 61 61 115 115 115 F F F 23 460-479 62 62 62 119 119 119 F F F 24 480-499 60 60 60 114 114 114 T T T 25 500-519 60 60 60 114 114 114 T T T 26 520-539 60 60 60 114 114 114 F F F 27 540-559 59 59 59 112 112 112 F F F 28 560-579 60 60 60 113 113 113 F F F 29 580-599 51 51 51 95 95 95 F F F Max gen: 62, Max flush: 119 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 120 120 120 F F F 1 020-039 63 63 63 120 120 120 F F F 2 040-059 63 63 63 120 120 120 F F F 3 060-079 63 63 63 120 120 120 F F F 4 080-099 62 62 62 118 118 118 F F F 5 100-119 63 63 63 120 120 120 F F F 6 120-139 54 54 54 103 103 103 F F F 7 140-159 50 50 50 94 94 94 F F F 8 160-179 52 52 52 99 99 99 F F F 9 180-199 61 61 61 115 115 115 F F F 10 200-219 54 54 54 104 104 104 F F F 11 220-239 61 61 61 115 115 115 F F F 12 240-259 60 60 60 114 114 114 F F F 13 260-279 60 60 60 114 114 114 F F F 14 280-299 62 62 62 117 117 117 F F F 15 300-319 63 63 63 120 120 120 F F F 16 320-339 59 59 59 112 112 112 F F F 17 340-359 59 59 59 112 112 112 F F F 18 360-379 63 63 63 120 120 120 F F F 19 380-399 63 63 63 120 120 120 F F F 20 400-419 62 62 62 117 117 117 F F F 21 420-439 62 62 62 117 117 117 F F F 22 440-459 61 61 61 115 115 115 F F F 23 460-479 62 62 62 119 119 119 F F F 24 480-499 63 63 63 120 120 120 F F F 25 500-519 63 63 63 120 120 120 F F F 26 520-539 60 60 60 114 114 114 F F F 27 540-559 59 59 59 112 112 112 F F F 28 560-579 60 60 60 113 113 113 F F F 29 580-599 51 51 51 95 95 95 F F F Max gen: 63, Max flush: 120 Begin loop 32 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.148458762Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:52:29.191324692Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.19170978Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.223332806Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.223874498Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.223909049Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.223923279Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.223937909Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443} {"msg":"Crucible 9badcb15-a16c-4c0f-bef4-989064375141 has session id: 83460357-ea98-4863-8c6a-9c979e5852a5","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.224314848Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"Upstairs opts: Upstairs UUID: 9badcb15-a16c-4c0f-bef4-989064375141, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.224360299Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"Crucible stats registered with UUID: 9badcb15-a16c-4c0f-bef4-989064375141","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.224385309Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.255891903Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.255923823Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443} {"msg":"9badcb15-a16c-4c0f-bef4-989064375141 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.256002435Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.256025616Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"client":"0","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.256054076Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"client":"0","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.256071727Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"client":"1","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.256101547Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"client":"1","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.256134168Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"client":"2","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.256166549Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"client":"2","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.256198459Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"":"io task","client":"2","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.25623242Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"":"io task","client":"0","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.256268181Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"":"io task","client":"1","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:38452)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.256308542Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"":"io task","client":"2","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:55417)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.256511456Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"":"io task","client":"0","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:59252)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.256553807Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"":"io task","client":"1","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.257071938Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"client":"2","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.257101129Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"client":"2","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.257117399Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"client":"1","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.25713311Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"client":"1","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.25714763Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"client":"0","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.25716191Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"client":"0","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.257689232Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"client":"2","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.257874586Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"client":"1","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.257985098Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"client":"0","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.258235413Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"client":"2","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.258275885Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"client":"2","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.258305505Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.258371047Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"client":"1","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.258400147Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"client":"1","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.258416177Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.258429138Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"client":"0","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.258443018Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"client":"0","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"[0]R flush_numbers[0..12]: [120, 120, 120, 120, 118, 120, 103, 94, 99, 115, 104, 115]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.258462549Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"":"downstairs","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"[0]R generation[0..12]: [63, 63, 63, 63, 62, 63, 54, 50, 52, 61, 54, 61]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.258476809Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"":"downstairs","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.258491019Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"":"downstairs","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"[1]R flush_numbers[0..12]: [120, 120, 120, 120, 118, 120, 103, 94, 99, 115, 104, 115]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.258504879Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"":"downstairs","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"[1]R generation[0..12]: [63, 63, 63, 63, 62, 63, 54, 50, 52, 61, 54, 61]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.25851942Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"":"downstairs","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.25853348Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"":"downstairs","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"[2]R flush_numbers[0..12]: [120, 120, 120, 120, 118, 120, 103, 94, 99, 115, 104, 115]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.25854741Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"":"downstairs","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"[2]R generation[0..12]: [63, 63, 63, 63, 62, 63, 54, 50, 52, 61, 54, 61]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.258561811Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"":"downstairs","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.258575821Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"":"downstairs","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.258592881Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"":"downstairs","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.258607492Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"":"downstairs","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"Next flush: 121","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:52:29.258620912Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"":"downstairs","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.258639872Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"":"downstairs","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.258653203Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.258665183Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"client":"0","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.258678353Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"client":"1","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.258691523Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"client":"2","":"downstairs","session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.258704754Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.258717154Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"9badcb15-a16c-4c0f-bef4-989064375141 is now active with session: 83460357-ea98-4863-8c6a-9c979e5852a5","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.258729894Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.258742104Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443,"session_id":"83460357-ea98-4863-8c6a-9c979e5852a5"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:29.258754575Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 321 len 32768 data: 3 3 5 5 6 6 7 7 02/30 Read block 91 len 36864 03/30 Read block 450 len 24576 04/30 Write block 355 len 8192 data: 5 5 05/30 Write block 146 len 32768 data: 5 7 7 8 7 7 8 5 06/30 Write block 523 len 40960 data: 5 7 6 9 7 5 5 3 4 4 07/30 Flush 08/30 Write block 314 len 8192 data: 6 6 09/30 Write block 223 len 12288 data: 5 4 3 10/30 Write block 547 len 12288 data: 6 6 6 11/30 Flush 12/30 Write block 419 len 8192 data: 3 3 13/30 Read block 323 len 4096 14/30 Read block 198 len 12288 15/30 Flush 16/30 Flush 17/30 Write block 169 len 4096 data: 5 18/30 Read block 258 len 24576 19/30 Flush 20/30 Write block 166 len 36864 data: 3 3 5 6 6 7 6 4 5 21/30 Write block 236 len 8192 data: 7 7 22/30 Read block 404 len 4096 23/30 Read block 40 len 8192 24/30 Write block 65 len 20480 data: 7 7 9 10 9 25/30 Read block 230 len 28672 26/30 Write block 545 len 4096 data: 5 27/30 Read block 58 len 16384 28/30 Write block 406 len 8192 data: 5 3 29/30 Read block 546 len 8192 30/30 Write block 91 len 12288 data: 8 7 7 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 9 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:52:29.276009169Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1443} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 120 120 120 F F F 1 020-039 63 63 63 120 120 120 F F F 2 040-059 63 63 63 120 120 120 F F F 3 060-079 63 63 63 120 120 120 T T T 4 080-099 62 62 62 118 118 118 T T T 5 100-119 63 63 63 120 120 120 F F F 6 120-139 54 54 54 103 103 103 F F F 7 140-159 64 64 64 121 121 121 F F F 8 160-179 64 64 64 125 125 125 T T T 9 180-199 61 61 61 115 115 115 F F F 10 200-219 54 54 54 104 104 104 F F F 11 220-239 64 64 64 122 122 122 T T T 12 240-259 60 60 60 114 114 114 F F F 13 260-279 60 60 60 114 114 114 F F F 14 280-299 62 62 62 117 117 117 F F F 15 300-319 64 64 64 122 122 122 F F F 16 320-339 64 64 64 121 121 121 F F F 17 340-359 64 64 64 121 121 121 F F F 18 360-379 63 63 63 120 120 120 F F F 19 380-399 63 63 63 120 120 120 F F F 20 400-419 64 64 64 123 123 123 T T T 21 420-439 64 64 64 123 123 123 F F F 22 440-459 61 61 61 115 115 115 F F F 23 460-479 62 62 62 119 119 119 F F F 24 480-499 63 63 63 120 120 120 F F F 25 500-519 63 63 63 120 120 120 F F F 26 520-539 64 64 64 121 121 121 F F F 27 540-559 64 64 64 122 122 122 T T T 28 560-579 60 60 60 113 113 113 F F F 29 580-599 51 51 51 95 95 95 F F F Max gen: 64, Max flush: 125 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 120 120 120 F F F 1 020-039 63 63 63 120 120 120 F F F 2 040-059 63 63 63 120 120 120 F F F 3 060-079 65 65 65 126 126 126 F F F 4 080-099 65 65 65 126 126 126 F F F 5 100-119 63 63 63 120 120 120 F F F 6 120-139 54 54 54 103 103 103 F F F 7 140-159 64 64 64 121 121 121 F F F 8 160-179 65 65 65 126 126 126 F F F 9 180-199 61 61 61 115 115 115 F F F 10 200-219 54 54 54 104 104 104 F F F 11 220-239 65 65 65 126 126 126 F F F 12 240-259 60 60 60 114 114 114 F F F 13 260-279 60 60 60 114 114 114 F F F 14 280-299 62 62 62 117 117 117 F F F 15 300-319 64 64 64 122 122 122 F F F 16 320-339 64 64 64 121 121 121 F F F 17 340-359 64 64 64 121 121 121 F F F 18 360-379 63 63 63 120 120 120 F F F 19 380-399 63 63 63 120 120 120 F F F 20 400-419 65 65 65 126 126 126 F F F 21 420-439 64 64 64 123 123 123 F F F 22 440-459 61 61 61 115 115 115 F F F 23 460-479 62 62 62 119 119 119 F F F 24 480-499 63 63 63 120 120 120 F F F 25 500-519 63 63 63 120 120 120 F F F 26 520-539 64 64 64 121 121 121 F F F 27 540-559 65 65 65 126 126 126 F F F 28 560-579 60 60 60 113 113 113 F F F 29 580-599 51 51 51 95 95 95 F F F Max gen: 65, Max flush: 126 Begin loop 33 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.574058116Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.617693103Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.618270266Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.618306876Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.618321537Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.618336217Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451} {"msg":"Crucible c29f320e-93b6-4246-b7b0-756af776b21c has session id: f8e72c74-0692-44df-92ca-af61f1f291d1","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.618828568Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"Upstairs opts: Upstairs UUID: c29f320e-93b6-4246-b7b0-756af776b21c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.618864829Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"Crucible stats registered with UUID: c29f320e-93b6-4246-b7b0-756af776b21c","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.618884989Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.649757329Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.64980183Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451} {"msg":"c29f320e-93b6-4246-b7b0-756af776b21c active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.649867331Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.649896802Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"client":"0","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.649923612Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"client":"0","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.649942743Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"client":"1","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.649969483Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"client":"1","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.649997324Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"client":"2","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.650080456Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"client":"2","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.650121617Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"":"io task","client":"2","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.650151627Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"":"io task","client":"0","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.650301361Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"":"io task","client":"1","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"ds_connection connected from Ok(127.0.0.1:45072)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.650341451Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"":"io task","client":"2","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"ds_connection connected from Ok(127.0.0.1:37499)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.650365182Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"":"io task","client":"1","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"ds_connection connected from Ok(127.0.0.1:56087)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.650389212Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"":"io task","client":"0","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.650833712Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"client":"2","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.650871003Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"client":"2","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.651099168Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"client":"1","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.651140499Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"client":"1","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.651167049Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"client":"0","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.651184Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"client":"0","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.651574888Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"client":"2","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.651603559Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"client":"0","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.651788703Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"client":"1","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.6521241Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"client":"0","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.652150771Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"client":"0","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.652165521Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.652246933Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"client":"1","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.652266003Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"client":"1","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.652279393Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.652295084Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"client":"2","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.652308944Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"client":"2","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"[0]R flush_numbers[0..12]: [120, 120, 120, 126, 126, 120, 103, 121, 126, 115, 104, 126]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.652322825Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"":"downstairs","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"[0]R generation[0..12]: [63, 63, 63, 65, 65, 63, 54, 64, 65, 61, 54, 65]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.652336625Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"":"downstairs","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.652350825Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"":"downstairs","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"[1]R flush_numbers[0..12]: [120, 120, 120, 126, 126, 120, 103, 121, 126, 115, 104, 126]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.652364695Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"":"downstairs","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"[1]R generation[0..12]: [63, 63, 63, 65, 65, 63, 54, 64, 65, 61, 54, 65]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.652378286Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"":"downstairs","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.652392196Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"":"downstairs","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"[2]R flush_numbers[0..12]: [120, 120, 120, 126, 126, 120, 103, 121, 126, 115, 104, 126]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.652405956Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"":"downstairs","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"[2]R generation[0..12]: [63, 63, 63, 65, 65, 63, 54, 64, 65, 61, 54, 65]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.652419736Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"":"downstairs","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.652433697Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"":"downstairs","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"Max found gen is 66","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2024-11-28T04:52:33.652455277Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"":"downstairs","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.652468868Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"":"downstairs","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"Next flush: 127","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.652481878Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"":"downstairs","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.652496698Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"":"downstairs","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.652510149Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.652522489Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"client":"0","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.652535919Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"client":"1","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.652549069Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"client":"2","":"downstairs","session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.65256221Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.65257467Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"c29f320e-93b6-4246-b7b0-756af776b21c is now active with session: f8e72c74-0692-44df-92ca-af61f1f291d1","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.6525875Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-28T04:52:33.65259982Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451,"session_id":"f8e72c74-0692-44df-92ca-af61f1f291d1"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:33.652613981Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 455 len 8192 data: 6 7 02/30 Read block 408 len 36864 03/30 Read block 72 len 4096 04/30 Write block 448 len 36864 data: 3 3 3 5 5 6 7 7 8 05/30 Write block 231 len 32768 data: 5 6 8 8 8 8 8 5 06/30 Read block 242 len 40960 07/30 Read block 307 len 36864 08/30 Read block 330 len 20480 09/30 Read block 555 len 8192 10/30 Flush 11/30 Flush 12/30 Write block 148 len 32768 data: 8 9 8 8 9 6 6 6 13/30 Read block 73 len 36864 14/30 Write block 454 len 40960 data: 8 8 9 7 6 8 7 8 8 8 15/30 Flush 16/30 Write block 266 len 32768 data: 6 6 6 8 11 10 10 9 17/30 Read block 239 len 8192 18/30 Read block 161 len 16384 19/30 Read block 342 len 12288 20/30 Read block 158 len 28672 21/30 Read block 7 len 32768 22/30 Read block 383 len 4096 23/30 Read block 392 len 8192 24/30 Write block 542 len 28672 data: 3 3 3 6 6 7 7 25/30 Read block 498 len 40960 26/30 Write block 506 len 20480 data: 7 8 7 7 6 27/30 Read block 37 len 24576 28/30 Write block 231 len 8192 data: 6 7 29/30 Read block 8 len 16384 30/30 Write block 35 len 40960 data: 4 5 5 6 8 8 7 7 6 7 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 8 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:52:33.673250528Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1451} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 120 120 120 F F F 1 020-039 63 63 63 120 120 120 T T T 2 040-059 63 63 63 120 120 120 T T T 3 060-079 65 65 65 126 126 126 F F F 4 080-099 65 65 65 126 126 126 F F F 5 100-119 63 63 63 120 120 120 F F F 6 120-139 54 54 54 103 103 103 F F F 7 140-159 66 66 66 129 129 129 F F F 8 160-179 65 65 65 126 126 126 F F F 9 180-199 61 61 61 115 115 115 F F F 10 200-219 54 54 54 104 104 104 F F F 11 220-239 66 66 66 127 127 127 T T T 12 240-259 60 60 60 114 114 114 F F F 13 260-279 60 60 60 114 114 114 T T T 14 280-299 62 62 62 117 117 117 F F F 15 300-319 64 64 64 122 122 122 F F F 16 320-339 64 64 64 121 121 121 F F F 17 340-359 64 64 64 121 121 121 F F F 18 360-379 63 63 63 120 120 120 F F F 19 380-399 63 63 63 120 120 120 F F F 20 400-419 65 65 65 126 126 126 F F F 21 420-439 64 64 64 123 123 123 F F F 22 440-459 66 66 66 129 129 129 F F F 23 460-479 66 66 66 129 129 129 F F F 24 480-499 63 63 63 120 120 120 F F F 25 500-519 63 63 63 120 120 120 T T T 26 520-539 64 64 64 121 121 121 F F F 27 540-559 65 65 65 126 126 126 T T T 28 560-579 60 60 60 113 113 113 F F F 29 580-599 51 51 51 95 95 95 F F F Max gen: 66, Max flush: 129 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 120 120 120 F F F 1 020-039 67 67 67 130 130 130 F F F 2 040-059 67 67 67 130 130 130 F F F 3 060-079 65 65 65 126 126 126 F F F 4 080-099 65 65 65 126 126 126 F F F 5 100-119 63 63 63 120 120 120 F F F 6 120-139 54 54 54 103 103 103 F F F 7 140-159 66 66 66 129 129 129 F F F 8 160-179 65 65 65 126 126 126 F F F 9 180-199 61 61 61 115 115 115 F F F 10 200-219 54 54 54 104 104 104 F F F 11 220-239 67 67 67 130 130 130 F F F 12 240-259 60 60 60 114 114 114 F F F 13 260-279 67 67 67 130 130 130 F F F 14 280-299 62 62 62 117 117 117 F F F 15 300-319 64 64 64 122 122 122 F F F 16 320-339 64 64 64 121 121 121 F F F 17 340-359 64 64 64 121 121 121 F F F 18 360-379 63 63 63 120 120 120 F F F 19 380-399 63 63 63 120 120 120 F F F 20 400-419 65 65 65 126 126 126 F F F 21 420-439 64 64 64 123 123 123 F F F 22 440-459 66 66 66 129 129 129 F F F 23 460-479 66 66 66 129 129 129 F F F 24 480-499 63 63 63 120 120 120 F F F 25 500-519 67 67 67 130 130 130 F F F 26 520-539 64 64 64 121 121 121 F F F 27 540-559 67 67 67 130 130 130 F F F 28 560-579 60 60 60 113 113 113 F F F 29 580-599 51 51 51 95 95 95 F F F Max gen: 67, Max flush: 130 Begin loop 34 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:37.944178617Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:37.988353576Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:37.988952619Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:37.98898976Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:37.98900468Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:37.98901949Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459} {"msg":"Crucible d909d377-3bb4-4e18-abee-56b374fff54f has session id: 245923e9-ab02-44f2-bea7-89d9ec581c55","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:37.989434449Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"Upstairs opts: Upstairs UUID: d909d377-3bb4-4e18-abee-56b374fff54f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:37.98947945Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"Crucible stats registered with UUID: d909d377-3bb4-4e18-abee-56b374fff54f","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:37.989504891Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.020243358Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.020282948Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459} {"msg":"d909d377-3bb4-4e18-abee-56b374fff54f active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.02035735Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.020381791Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"client":"0","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.020412701Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"client":"0","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.020431702Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"client":"1","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.020449452Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"client":"1","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.020512433Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"client":"2","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.020532364Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"client":"2","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.020548154Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"":"io task","client":"2","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.020563675Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"":"io task","client":"0","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.020579045Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"":"io task","client":"1","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"ds_connection connected from Ok(127.0.0.1:54524)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.020851721Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"":"io task","client":"2","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"ds_connection connected from Ok(127.0.0.1:63662)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.020892102Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"":"io task","client":"0","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"ds_connection connected from Ok(127.0.0.1:41252)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.020911412Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"":"io task","client":"1","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.021580137Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"client":"2","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.021620837Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"client":"2","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.021657578Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"client":"1","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.02172587Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"client":"1","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.021768371Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"client":"0","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.021799021Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"client":"0","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.022306183Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"client":"1","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.022347693Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"client":"0","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.022375994Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"client":"2","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.022808673Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"client":"1","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.022833774Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"client":"1","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.022848564Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.022913376Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"client":"0","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.022942636Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"client":"0","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.022958846Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.022973937Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"client":"2","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.022988707Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"client":"2","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"[0]R flush_numbers[0..12]: [120, 130, 130, 126, 126, 120, 103, 129, 126, 115, 104, 130]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.023007348Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"":"downstairs","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"[0]R generation[0..12]: [63, 67, 67, 65, 65, 63, 54, 66, 65, 61, 54, 67]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.023023298Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"":"downstairs","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.023039318Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"":"downstairs","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"[1]R flush_numbers[0..12]: [120, 130, 130, 126, 126, 120, 103, 129, 126, 115, 104, 130]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.023054769Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"":"downstairs","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"[1]R generation[0..12]: [63, 67, 67, 65, 65, 63, 54, 66, 65, 61, 54, 67]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.023069839Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"":"downstairs","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.023085169Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"":"downstairs","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"[2]R flush_numbers[0..12]: [120, 130, 130, 126, 126, 120, 103, 129, 126, 115, 104, 130]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.02310035Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"":"downstairs","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"[2]R generation[0..12]: [63, 67, 67, 65, 65, 63, 54, 66, 65, 61, 54, 67]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.02311785Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"":"downstairs","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.023145001Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"":"downstairs","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2024-11-28T04:52:38.023172671Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"":"downstairs","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.023208742Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"":"downstairs","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"Next flush: 131","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.023235403Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"":"downstairs","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.023261833Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"":"downstairs","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.023285684Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.023308784Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"client":"0","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.023339945Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"client":"1","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.023362045Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"client":"2","":"downstairs","session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.023382356Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 crucible","level":30,"time":"2024-11-28T04:52:38.023408116Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"d909d377-3bb4-4e18-abee-56b374fff54f is now active with session: 245923e9-ab02-44f2-bea7-89d9ec581c55","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.023431717Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.023453037Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459,"session_id":"245923e9-ab02-44f2-bea7-89d9ec581c55"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:38.023475578Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 208 len 16384 data: 6 5 9 9 02/30 Read block 420 len 4096 03/30 Write block 165 len 36864 data: 3 4 4 6 7 7 8 7 5 04/30 Write block 492 len 36864 data: 5 4 6 7 6 6 7 7 11 05/30 Read block 550 len 36864 06/30 Read block 590 len 32768 07/30 Write block 556 len 8192 data: 9 8 08/30 Read block 289 len 36864 09/30 Read block 193 len 28672 10/30 Read block 257 len 16384 11/30 Write block 426 len 20480 data: 5 5 7 6 5 12/30 Read block 334 len 40960 13/30 Flush 14/30 Write block 197 len 12288 data: 8 8 7 15/30 Write block 128 len 8192 data: 6 4 16/30 Write block 195 len 16384 data: 9 7 9 9 17/30 Write block 469 len 12288 data: 6 5 4 18/30 Read block 27 len 16384 19/30 Read block 22 len 20480 20/30 Read block 223 len 40960 21/30 Read block 364 len 32768 22/30 Flush 23/30 Write block 336 len 16384 data: 9 7 9 8 24/30 Flush 25/30 Write block 462 len 8192 data: 9 9 26/30 Read block 373 len 40960 27/30 Read block 394 len 24576 28/30 Read block 261 len 40960 29/30 Read block 181 len 16384 30/30 Write block 528 len 32768 data: 6 6 4 5 5 5 6 5 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 2 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:52:38.042982851Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1459} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 120 120 120 F F F 1 020-039 67 67 67 130 130 130 F F F 2 040-059 67 67 67 130 130 130 F F F 3 060-079 65 65 65 126 126 126 F F F 4 080-099 65 65 65 126 126 126 F F F 5 100-119 63 63 63 120 120 120 F F F 6 120-139 68 68 68 132 132 132 F F F 7 140-159 66 66 66 129 129 129 F F F 8 160-179 68 68 68 131 131 131 F F F 9 180-199 68 68 68 132 132 132 F F F 10 200-219 68 68 68 131 131 131 F F F 11 220-239 67 67 67 130 130 130 F F F 12 240-259 60 60 60 114 114 114 F F F 13 260-279 67 67 67 130 130 130 F F F 14 280-299 62 62 62 117 117 117 F F F 15 300-319 64 64 64 122 122 122 F F F 16 320-339 68 68 68 133 133 133 F F F 17 340-359 64 64 64 121 121 121 F F F 18 360-379 63 63 63 120 120 120 F F F 19 380-399 63 63 63 120 120 120 F F F 20 400-419 65 65 65 126 126 126 F F F 21 420-439 68 68 68 131 131 131 F F F 22 440-459 66 66 66 129 129 129 F F F 23 460-479 68 68 68 132 132 132 T T T 24 480-499 68 68 68 131 131 131 F F F 25 500-519 68 68 68 131 131 131 F F F 26 520-539 64 64 64 121 121 121 T T T 27 540-559 68 68 68 131 131 131 F F F 28 560-579 60 60 60 113 113 113 F F F 29 580-599 51 51 51 95 95 95 F F F Max gen: 68, Max flush: 133 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 120 120 120 F F F 1 020-039 67 67 67 130 130 130 F F F 2 040-059 67 67 67 130 130 130 F F F 3 060-079 65 65 65 126 126 126 F F F 4 080-099 65 65 65 126 126 126 F F F 5 100-119 63 63 63 120 120 120 F F F 6 120-139 68 68 68 132 132 132 F F F 7 140-159 66 66 66 129 129 129 F F F 8 160-179 68 68 68 131 131 131 F F F 9 180-199 68 68 68 132 132 132 F F F 10 200-219 68 68 68 131 131 131 F F F 11 220-239 67 67 67 130 130 130 F F F 12 240-259 60 60 60 114 114 114 F F F 13 260-279 67 67 67 130 130 130 F F F 14 280-299 62 62 62 117 117 117 F F F 15 300-319 64 64 64 122 122 122 F F F 16 320-339 68 68 68 133 133 133 F F F 17 340-359 64 64 64 121 121 121 F F F 18 360-379 63 63 63 120 120 120 F F F 19 380-399 63 63 63 120 120 120 F F F 20 400-419 65 65 65 126 126 126 F F F 21 420-439 68 68 68 131 131 131 F F F 22 440-459 66 66 66 129 129 129 F F F 23 460-479 69 69 69 134 134 134 F F F 24 480-499 68 68 68 131 131 131 F F F 25 500-519 68 68 68 131 131 131 F F F 26 520-539 69 69 69 134 134 134 F F F 27 540-559 68 68 68 131 131 131 F F F 28 560-579 60 60 60 113 113 113 F F F 29 580-599 51 51 51 95 95 95 F F F Max gen: 69, Max flush: 134 Begin loop 35 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.296179805Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:52:42.338727568Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.339089036Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.370681622Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.371264664Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.371299975Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.371314046Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.371328276Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467} {"msg":"Crucible dd6d8a12-f0f4-4580-8340-a5a2c4f87566 has session id: f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.371751305Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"Upstairs opts: Upstairs UUID: dd6d8a12-f0f4-4580-8340-a5a2c4f87566, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.371795606Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"Crucible stats registered with UUID: dd6d8a12-f0f4-4580-8340-a5a2c4f87566","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.371813156Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.402465761Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.402491262Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467} {"msg":"dd6d8a12-f0f4-4580-8340-a5a2c4f87566 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.402570364Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.402591094Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"client":"0","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.402618625Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"client":"0","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.402633665Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"client":"1","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.402647605Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"client":"1","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.402660875Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"client":"2","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.402675276Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"client":"2","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.402688466Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"":"io task","client":"2","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.402702756Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"":"io task","client":"0","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.402792078Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"":"io task","client":"1","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"ds_connection connected from Ok(127.0.0.1:34327)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.402827919Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"":"io task","client":"2","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"ds_connection connected from Ok(127.0.0.1:57112)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.40285368Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"":"io task","client":"0","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"ds_connection connected from Ok(127.0.0.1:40142)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.402950502Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"":"io task","client":"1","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.403624226Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"client":"1","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.403653677Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"client":"1","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.403669227Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"client":"2","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.403683408Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"client":"2","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.403700768Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"client":"0","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.403714498Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"client":"0","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.40425345Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"client":"1","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.404342472Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"client":"0","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.404554447Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"client":"2","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.404748731Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"client":"1","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.404790372Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"client":"1","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.404821402Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.404881784Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"client":"0","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.404910584Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"client":"0","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.404926464Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.404942185Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"client":"2","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.404956165Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"client":"2","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"[0]R flush_numbers[0..12]: [120, 130, 130, 126, 126, 120, 132, 129, 131, 132, 131, 130]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.404984426Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"":"downstairs","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"[0]R generation[0..12]: [63, 67, 67, 65, 65, 63, 68, 66, 68, 68, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.404999226Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"":"downstairs","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.405017126Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"":"downstairs","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"[1]R flush_numbers[0..12]: [120, 130, 130, 126, 126, 120, 132, 129, 131, 132, 131, 130]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.405031657Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"":"downstairs","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"[1]R generation[0..12]: [63, 67, 67, 65, 65, 63, 68, 66, 68, 68, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.405045527Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"":"downstairs","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.405059738Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"":"downstairs","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"[2]R flush_numbers[0..12]: [120, 130, 130, 126, 126, 120, 132, 129, 131, 132, 131, 130]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.405073748Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"":"downstairs","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"[2]R generation[0..12]: [63, 67, 67, 65, 65, 63, 68, 66, 68, 68, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.405087988Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"":"downstairs","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.405102628Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"":"downstairs","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.405116119Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"":"downstairs","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.405129889Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"":"downstairs","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"Next flush: 135","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:52:42.405143919Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"":"downstairs","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.40516437Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"":"downstairs","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.40517872Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.4051911Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"client":"0","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.405204671Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"client":"1","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.405218031Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"client":"2","":"downstairs","session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.405234231Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.405246971Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"dd6d8a12-f0f4-4580-8340-a5a2c4f87566 is now active with session: f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.405259922Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.405275302Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467,"session_id":"f23ab9ae-04e4-424a-a4a4-4317f3e7cd7a"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:42.405288332Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 182 len 24576 data: 4 7 8 8 10 10 03/30 Write block 307 len 8192 data: 4 4 04/30 Write block 357 len 28672 data: 5 7 6 5 6 5 5 05/30 Write block 582 len 32768 data: 3 7 7 7 7 5 6 6 06/30 Write block 100 len 28672 data: 5 6 7 7 6 5 4 07/30 Read block 31 len 20480 08/30 Read block 106 len 40960 09/30 Flush 10/30 Write block 299 len 16384 data: 5 6 6 6 11/30 Write block 184 len 12288 data: 9 9 11 12/30 Write block 448 len 36864 data: 4 4 4 6 6 7 9 9 10 13/30 Read block 300 len 40960 14/30 Read block 388 len 32768 15/30 Read block 440 len 36864 16/30 Read block 333 len 20480 17/30 Read block 155 len 36864 18/30 Read block 45 len 8192 19/30 Read block 77 len 16384 20/30 Write block 448 len 16384 data: 5 5 5 7 21/30 Read block 437 len 16384 22/30 Flush 23/30 Write block 334 len 28672 data: 9 10 10 8 10 9 8 24/30 Write block 18 len 28672 data: 7 10 9 10 8 10 9 25/30 Flush 26/30 Write block 347 len 28672 data: 7 6 5 6 7 7 4 27/30 Read block 356 len 28672 28/30 Read block 389 len 4096 29/30 Write block 549 len 8192 data: 7 7 30/30 Read block 137 len 40960 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 7 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 10 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 4 4 5 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:52:42.424594281Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1467} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 138 138 138 F F F 1 020-039 70 70 70 138 138 138 F F F 2 040-059 67 67 67 130 130 130 F F F 3 060-079 65 65 65 126 126 126 F F F 4 080-099 65 65 65 126 126 126 F F F 5 100-119 70 70 70 136 136 136 F F F 6 120-139 68 68 68 132 132 132 F F F 7 140-159 66 66 66 129 129 129 F F F 8 160-179 68 68 68 131 131 131 F F F 9 180-199 70 70 70 137 137 137 F F F 10 200-219 68 68 68 131 131 131 F F F 11 220-239 67 67 67 130 130 130 F F F 12 240-259 60 60 60 114 114 114 F F F 13 260-279 67 67 67 130 130 130 F F F 14 280-299 70 70 70 137 137 137 F F F 15 300-319 70 70 70 137 137 137 F F F 16 320-339 70 70 70 138 138 138 F F F 17 340-359 70 70 70 138 138 138 T T T 18 360-379 70 70 70 136 136 136 F F F 19 380-399 63 63 63 120 120 120 F F F 20 400-419 65 65 65 126 126 126 F F F 21 420-439 68 68 68 131 131 131 F F F 22 440-459 70 70 70 137 137 137 F F F 23 460-479 69 69 69 134 134 134 F F F 24 480-499 68 68 68 131 131 131 F F F 25 500-519 68 68 68 131 131 131 F F F 26 520-539 69 69 69 134 134 134 F F F 27 540-559 68 68 68 131 131 131 T T T 28 560-579 60 60 60 113 113 113 F F F 29 580-599 70 70 70 136 136 136 F F F Max gen: 70, Max flush: 138 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 138 138 138 F F F 1 020-039 70 70 70 138 138 138 F F F 2 040-059 67 67 67 130 130 130 F F F 3 060-079 65 65 65 126 126 126 F F F 4 080-099 65 65 65 126 126 126 F F F 5 100-119 70 70 70 136 136 136 F F F 6 120-139 68 68 68 132 132 132 F F F 7 140-159 66 66 66 129 129 129 F F F 8 160-179 68 68 68 131 131 131 F F F 9 180-199 70 70 70 137 137 137 F F F 10 200-219 68 68 68 131 131 131 F F F 11 220-239 67 67 67 130 130 130 F F F 12 240-259 60 60 60 114 114 114 F F F 13 260-279 67 67 67 130 130 130 F F F 14 280-299 70 70 70 137 137 137 F F F 15 300-319 70 70 70 137 137 137 F F F 16 320-339 70 70 70 138 138 138 F F F 17 340-359 71 71 71 139 139 139 F F F 18 360-379 70 70 70 136 136 136 F F F 19 380-399 63 63 63 120 120 120 F F F 20 400-419 65 65 65 126 126 126 F F F 21 420-439 68 68 68 131 131 131 F F F 22 440-459 70 70 70 137 137 137 F F F 23 460-479 69 69 69 134 134 134 F F F 24 480-499 68 68 68 131 131 131 F F F 25 500-519 68 68 68 131 131 131 F F F 26 520-539 69 69 69 134 134 134 F F F 27 540-559 71 71 71 139 139 139 F F F 28 560-579 60 60 60 113 113 113 F F F 29 580-599 70 70 70 136 136 136 F F F Max gen: 71, Max flush: 139 Begin loop 36 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.705878025Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.750491193Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.75126804Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.751306951Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.751321811Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.751336451Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475} {"msg":"Crucible b2cad1f4-506d-44fa-8ee1-849315c6fc6b has session id: 4035ff0c-0864-4b8e-b043-ca622562ab82","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.75172035Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"Upstairs opts: Upstairs UUID: b2cad1f4-506d-44fa-8ee1-849315c6fc6b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.751766121Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"Crucible stats registered with UUID: b2cad1f4-506d-44fa-8ee1-849315c6fc6b","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.751792101Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.783074879Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.78310695Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475} {"msg":"b2cad1f4-506d-44fa-8ee1-849315c6fc6b active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.783181502Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.783203692Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"client":"0","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.783233723Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"client":"0","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.783251673Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"client":"1","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.783268374Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"client":"1","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.783288854Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"client":"2","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.783326165Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"client":"2","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.783360765Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"":"io task","client":"2","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.783395956Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"":"io task","client":"0","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.783433627Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"":"io task","client":"1","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"ds_connection connected from Ok(127.0.0.1:49891)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.78358306Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"":"io task","client":"2","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"ds_connection connected from Ok(127.0.0.1:50366)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.783631381Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"":"io task","client":"0","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"ds_connection connected from Ok(127.0.0.1:45529)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.783664252Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"":"io task","client":"1","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.784241035Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"client":"1","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.784287196Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"client":"1","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.784317686Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"client":"0","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.784345147Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"client":"0","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.784360397Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"client":"2","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.784374227Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"client":"2","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.784871798Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"client":"1","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.784900509Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"client":"0","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.785069813Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"client":"2","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.785457421Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"client":"1","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.785492392Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"client":"1","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.785509692Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.785528663Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"client":"0","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.785554873Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"client":"0","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.785569773Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.785595634Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"client":"2","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.785609674Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"client":"2","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"[0]R flush_numbers[0..12]: [138, 138, 130, 126, 126, 136, 132, 129, 131, 137, 131, 130]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.785648555Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"":"downstairs","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"[0]R generation[0..12]: [70, 70, 67, 65, 65, 70, 68, 66, 68, 70, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.785665775Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"":"downstairs","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.785680276Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"":"downstairs","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"[1]R flush_numbers[0..12]: [138, 138, 130, 126, 126, 136, 132, 129, 131, 137, 131, 130]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.785694266Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"":"downstairs","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"[1]R generation[0..12]: [70, 70, 67, 65, 65, 70, 68, 66, 68, 70, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.785707856Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"":"downstairs","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.785721897Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"":"downstairs","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"[2]R flush_numbers[0..12]: [138, 138, 130, 126, 126, 136, 132, 129, 131, 137, 131, 130]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.785738447Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"":"downstairs","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"[2]R generation[0..12]: [70, 70, 67, 65, 65, 70, 68, 66, 68, 70, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.785752217Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"":"downstairs","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.785766168Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"":"downstairs","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.785779188Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"":"downstairs","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.785792578Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"":"downstairs","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"Next flush: 140","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.785805618Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"":"downstairs","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:52:46.785818709Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"":"downstairs","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.785839899Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.785852199Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"client":"0","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.78586611Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"client":"1","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.78587935Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"client":"2","":"downstairs","session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.785892751Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.785905111Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"b2cad1f4-506d-44fa-8ee1-849315c6fc6b is now active with session: 4035ff0c-0864-4b8e-b043-ca622562ab82","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.785918031Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:46.785930151Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475,"session_id":"4035ff0c-0864-4b8e-b043-ca622562ab82"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-28T04:52:46.785944481Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 347 len 28672 data: 8 7 6 7 8 8 5 02/30 Read block 323 len 16384 03/30 Read block 549 len 16384 04/30 Read block 380 len 32768 05/30 Write block 149 len 40960 data: 10 9 9 10 7 7 7 6 6 6 06/30 Write block 173 len 32768 data: 6 6 5 5 4 3 3 3 07/30 Read block 355 len 8192 08/30 Read block 375 len 4096 09/30 Write block 294 len 24576 data: 4 3 4 4 5 6 10/30 Read block 152 len 40960 11/30 Read block 144 len 40960 12/30 Write block 408 len 40960 data: 5 5 6 4 4 5 5 5 3 3 13/30 Write block 341 len 40960 data: 7 7 7 8 8 7 9 8 7 8 14/30 Flush 15/30 Write block 69 len 32768 data: 10 9 10 9 11 11 9 6 16/30 Read block 105 len 40960 17/30 Write block 89 len 16384 data: 8 7 9 8 18/30 Read block 456 len 28672 19/30 Read block 410 len 12288 20/30 Read block 264 len 12288 21/30 Read block 332 len 24576 22/30 Read block 237 len 16384 23/30 Flush 24/30 Write block 248 len 32768 data: 7 7 8 7 6 7 7 7 25/30 Write block 267 len 36864 data: 7 7 9 12 11 11 10 9 7 26/30 Write block 404 len 20480 data: 6 5 6 4 6 27/30 Write block 412 len 24576 data: 5 6 6 6 4 4 28/30 Read block 112 len 32768 29/30 Write block 476 len 4096 data: 8 30/30 Write block 246 len 28672 data: 8 7 8 8 9 8 7 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 8 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:52:46.806667931Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1475} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 138 138 138 F F F 1 020-039 70 70 70 138 138 138 F F F 2 040-059 67 67 67 130 130 130 F F F 3 060-079 72 72 72 141 141 141 F F F 4 080-099 72 72 72 141 141 141 F F F 5 100-119 70 70 70 136 136 136 F F F 6 120-139 68 68 68 132 132 132 F F F 7 140-159 72 72 72 140 140 140 F F F 8 160-179 72 72 72 140 140 140 F F F 9 180-199 72 72 72 140 140 140 F F F 10 200-219 68 68 68 131 131 131 F F F 11 220-239 67 67 67 130 130 130 F F F 12 240-259 60 60 60 114 114 114 T T T 13 260-279 67 67 67 130 130 130 T T T 14 280-299 72 72 72 140 140 140 F F F 15 300-319 70 70 70 137 137 137 F F F 16 320-339 70 70 70 138 138 138 F F F 17 340-359 72 72 72 140 140 140 F F F 18 360-379 70 70 70 136 136 136 F F F 19 380-399 63 63 63 120 120 120 F F F 20 400-419 72 72 72 140 140 140 T T T 21 420-439 68 68 68 131 131 131 F F F 22 440-459 70 70 70 137 137 137 F F F 23 460-479 69 69 69 134 134 134 T T T 24 480-499 68 68 68 131 131 131 F F F 25 500-519 68 68 68 131 131 131 F F F 26 520-539 69 69 69 134 134 134 F F F 27 540-559 71 71 71 139 139 139 F F F 28 560-579 60 60 60 113 113 113 F F F 29 580-599 70 70 70 136 136 136 F F F Max gen: 72, Max flush: 141 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 138 138 138 F F F 1 020-039 70 70 70 138 138 138 F F F 2 040-059 67 67 67 130 130 130 F F F 3 060-079 72 72 72 141 141 141 F F F 4 080-099 72 72 72 141 141 141 F F F 5 100-119 70 70 70 136 136 136 F F F 6 120-139 68 68 68 132 132 132 F F F 7 140-159 72 72 72 140 140 140 F F F 8 160-179 72 72 72 140 140 140 F F F 9 180-199 72 72 72 140 140 140 F F F 10 200-219 68 68 68 131 131 131 F F F 11 220-239 67 67 67 130 130 130 F F F 12 240-259 73 73 73 142 142 142 F F F 13 260-279 73 73 73 142 142 142 F F F 14 280-299 72 72 72 140 140 140 F F F 15 300-319 70 70 70 137 137 137 F F F 16 320-339 70 70 70 138 138 138 F F F 17 340-359 72 72 72 140 140 140 F F F 18 360-379 70 70 70 136 136 136 F F F 19 380-399 63 63 63 120 120 120 F F F 20 400-419 73 73 73 142 142 142 F F F 21 420-439 68 68 68 131 131 131 F F F 22 440-459 70 70 70 137 137 137 F F F 23 460-479 73 73 73 142 142 142 F F F 24 480-499 68 68 68 131 131 131 F F F 25 500-519 68 68 68 131 131 131 F F F 26 520-539 69 69 69 134 134 134 F F F 27 540-559 71 71 71 139 139 139 F F F 28 560-579 60 60 60 113 113 113 F F F 29 580-599 70 70 70 136 136 136 F F F Max gen: 73, Max flush: 142 Begin loop 37 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.065450577Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.109440482Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.110074196Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.110123317Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.110140167Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.110159747Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483} {"msg":"Crucible dd37aad8-ac08-47c0-ae88-5393839a5b4d has session id: aeee1c40-39d4-4e7c-84e7-802722042338","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.110565116Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"Upstairs opts: Upstairs UUID: dd37aad8-ac08-47c0-ae88-5393839a5b4d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.110611987Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"Crucible stats registered with UUID: dd37aad8-ac08-47c0-ae88-5393839a5b4d","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.110635588Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.141463286Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.141488967Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483} {"msg":"dd37aad8-ac08-47c0-ae88-5393839a5b4d active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.141556788Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.141585359Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"client":"0","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.14161333Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"client":"0","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.14164032Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"client":"1","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.141667841Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"client":"1","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.141697702Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"client":"2","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.141725622Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"client":"2","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.141751533Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"":"io task","client":"2","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.141780773Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"":"io task","client":"0","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.141809684Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"":"io task","client":"1","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"ds_connection connected from Ok(127.0.0.1:41127)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.141835954Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"":"io task","client":"2","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"ds_connection connected from Ok(127.0.0.1:44913)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.141862755Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"":"io task","client":"0","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"ds_connection connected from Ok(127.0.0.1:54812)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.141887986Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"":"io task","client":"1","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.142526989Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"client":"2","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.14255742Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"client":"2","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.142666552Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"client":"1","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.142711424Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"client":"1","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.142766535Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"client":"0","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.142789955Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"client":"0","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.143190204Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"client":"2","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.143221255Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"client":"0","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.143323677Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"client":"1","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.143740386Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"client":"0","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.143767806Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"client":"0","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.143787027Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.143844418Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"client":"1","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.143873079Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"client":"1","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.143889719Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.143904069Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"client":"2","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.14391897Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"client":"2","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"[0]R flush_numbers[0..12]: [138, 138, 130, 141, 141, 136, 132, 140, 140, 140, 131, 130]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.14393458Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"":"downstairs","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"[0]R generation[0..12]: [70, 70, 67, 72, 72, 70, 68, 72, 72, 72, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.14395034Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"":"downstairs","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.143966001Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"":"downstairs","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"[1]R flush_numbers[0..12]: [138, 138, 130, 141, 141, 136, 132, 140, 140, 140, 131, 130]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.143981081Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"":"downstairs","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"[1]R generation[0..12]: [70, 70, 67, 72, 72, 70, 68, 72, 72, 72, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.143996001Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"":"downstairs","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.144011652Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"":"downstairs","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"[2]R flush_numbers[0..12]: [138, 138, 130, 141, 141, 136, 132, 140, 140, 140, 131, 130]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.144026732Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"":"downstairs","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"[2]R generation[0..12]: [70, 70, 67, 72, 72, 70, 68, 72, 72, 72, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.144041692Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"":"downstairs","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.144056833Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"":"downstairs","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.144075063Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"":"downstairs","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.144090253Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"":"downstairs","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"Next flush: 143","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2024-11-28T04:52:51.144104904Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"":"downstairs","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.144130454Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"":"downstairs","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.144144975Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.144158445Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"client":"0","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.144173075Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"client":"1","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.144187645Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"client":"2","":"downstairs","session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.144202566Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.144216256Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"dd37aad8-ac08-47c0-ae88-5393839a5b4d is now active with session: aeee1c40-39d4-4e7c-84e7-802722042338","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.144230466Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.144244027Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483,"session_id":"aeee1c40-39d4-4e7c-84e7-802722042338"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:51.144257897Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 221 len 28672 data: 6 6 6 5 4 2 5 02/30 Flush 03/30 Flush 04/30 Write block 240 len 20480 data: 7 6 5 6 6 05/30 Flush 06/30 Write block 523 len 16384 data: 6 8 7 10 07/30 Read block 316 len 36864 08/30 Read block 419 len 4096 09/30 Flush 10/30 Write block 366 len 20480 data: 5 5 6 6 6 11/30 Write block 258 len 4096 data: 7 12/30 Write block 259 len 4096 data: 6 13/30 Flush 14/30 Write block 359 len 20480 data: 7 6 7 6 6 15/30 Flush 16/30 Write block 355 len 20480 data: 6 6 6 8 8 17/30 Read block 326 len 24576 18/30 Write block 175 len 20480 data: 6 6 5 4 4 19/30 Read block 516 len 8192 20/30 Read block 366 len 28672 21/30 Flush 22/30 Write block 74 len 24576 data: 12 10 7 7 6 6 23/30 Read block 128 len 40960 24/30 Write block 283 len 32768 data: 4 6 8 8 7 5 5 5 25/30 Read block 221 len 36864 26/30 Read block 377 len 12288 27/30 Write block 71 len 8192 data: 11 10 28/30 Read block 76 len 20480 29/30 Read block 348 len 8192 30/30 Write block 387 len 12288 data: 7 7 7 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 6 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:52:51.163040154Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1483} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 138 138 138 F F F 1 020-039 70 70 70 138 138 138 F F F 2 040-059 67 67 67 130 130 130 F F F 3 060-079 72 72 72 141 141 141 T T T 4 080-099 72 72 72 141 141 141 F F F 5 100-119 70 70 70 136 136 136 F F F 6 120-139 68 68 68 132 132 132 F F F 7 140-159 72 72 72 140 140 140 F F F 8 160-179 74 74 74 149 149 149 F F F 9 180-199 72 72 72 140 140 140 F F F 10 200-219 68 68 68 131 131 131 F F F 11 220-239 74 74 74 143 143 143 F F F 12 240-259 74 74 74 147 147 147 F F F 13 260-279 73 73 73 142 142 142 F F F 14 280-299 72 72 72 140 140 140 T T T 15 300-319 70 70 70 137 137 137 F F F 16 320-339 70 70 70 138 138 138 F F F 17 340-359 74 74 74 149 149 149 F F F 18 360-379 74 74 74 148 148 148 F F F 19 380-399 63 63 63 120 120 120 T T T 20 400-419 73 73 73 142 142 142 F F F 21 420-439 68 68 68 131 131 131 F F F 22 440-459 70 70 70 137 137 137 F F F 23 460-479 73 73 73 142 142 142 F F F 24 480-499 68 68 68 131 131 131 F F F 25 500-519 68 68 68 131 131 131 F F F 26 520-539 74 74 74 146 146 146 F F F 27 540-559 71 71 71 139 139 139 F F F 28 560-579 60 60 60 113 113 113 F F F 29 580-599 70 70 70 136 136 136 F F F Max gen: 74, Max flush: 149 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 138 138 138 F F F 1 020-039 70 70 70 138 138 138 F F F 2 040-059 67 67 67 130 130 130 F F F 3 060-079 75 75 75 150 150 150 F F F 4 080-099 72 72 72 141 141 141 F F F 5 100-119 70 70 70 136 136 136 F F F 6 120-139 68 68 68 132 132 132 F F F 7 140-159 72 72 72 140 140 140 F F F 8 160-179 74 74 74 149 149 149 F F F 9 180-199 72 72 72 140 140 140 F F F 10 200-219 68 68 68 131 131 131 F F F 11 220-239 74 74 74 143 143 143 F F F 12 240-259 74 74 74 147 147 147 F F F 13 260-279 73 73 73 142 142 142 F F F 14 280-299 75 75 75 150 150 150 F F F 15 300-319 70 70 70 137 137 137 F F F 16 320-339 70 70 70 138 138 138 F F F 17 340-359 74 74 74 149 149 149 F F F 18 360-379 74 74 74 148 148 148 F F F 19 380-399 75 75 75 150 150 150 F F F 20 400-419 73 73 73 142 142 142 F F F 21 420-439 68 68 68 131 131 131 F F F 22 440-459 70 70 70 137 137 137 F F F 23 460-479 73 73 73 142 142 142 F F F 24 480-499 68 68 68 131 131 131 F F F 25 500-519 68 68 68 131 131 131 F F F 26 520-539 74 74 74 146 146 146 F F F 27 540-559 71 71 71 139 139 139 F F F 28 560-579 60 60 60 113 113 113 F F F 29 580-599 70 70 70 136 136 136 F F F Max gen: 75, Max flush: 150 Begin loop 38 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.419991791Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:52:55.46237068Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.462731768Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.494524628Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.495070939Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.49510565Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.49511981Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.495134381Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491} {"msg":"Crucible e0032ecb-ba2c-445e-8c21-f7edbbda7832 has session id: 27329231-dabf-47ae-a995-66e7d2befb72","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.495508349Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"Upstairs opts: Upstairs UUID: e0032ecb-ba2c-445e-8c21-f7edbbda7832, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.49555623Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"Crucible stats registered with UUID: e0032ecb-ba2c-445e-8c21-f7edbbda7832","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.49558133Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.526322397Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.526348178Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491} {"msg":"e0032ecb-ba2c-445e-8c21-f7edbbda7832 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.52642286Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.52644159Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"client":"0","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.52646436Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"client":"0","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.526478761Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"client":"1","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.526492551Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"client":"1","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.526505921Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"client":"2","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.526520322Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"client":"2","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.526533652Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"":"io task","client":"2","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.526548302Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"":"io task","client":"0","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.526571723Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"":"io task","client":"1","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"ds_connection connected from Ok(127.0.0.1:56837)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.526599283Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"":"io task","client":"2","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"ds_connection connected from Ok(127.0.0.1:48538)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.526633234Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"":"io task","client":"0","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"ds_connection connected from Ok(127.0.0.1:54628)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.526663555Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"":"io task","client":"1","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.52736646Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"client":"1","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.527393961Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"client":"1","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.527409001Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"client":"2","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.527422791Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"client":"2","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.527486712Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"client":"0","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.527502623Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"client":"0","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.528022754Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"client":"2","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.528154167Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"client":"1","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.528203548Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"client":"0","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.528529055Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"client":"2","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.528558166Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"client":"2","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.528574946Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.52876993Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"client":"1","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.528812811Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"client":"1","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.528838302Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.528860592Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"client":"0","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.528885083Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"client":"0","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"[0]R flush_numbers[0..12]: [138, 138, 130, 150, 141, 136, 132, 140, 149, 140, 131, 143]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.528910813Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"":"downstairs","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"[0]R generation[0..12]: [70, 70, 67, 75, 72, 70, 68, 72, 74, 72, 68, 74]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.528936024Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"":"downstairs","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.528960244Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"":"downstairs","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"[1]R flush_numbers[0..12]: [138, 138, 130, 150, 141, 136, 132, 140, 149, 140, 131, 143]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.528985445Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"":"downstairs","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"[1]R generation[0..12]: [70, 70, 67, 75, 72, 70, 68, 72, 74, 72, 68, 74]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.529012306Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"":"downstairs","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.529039896Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"":"downstairs","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"[2]R flush_numbers[0..12]: [138, 138, 130, 150, 141, 136, 132, 140, 149, 140, 131, 143]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.529066337Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"":"downstairs","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"[2]R generation[0..12]: [70, 70, 67, 75, 72, 70, 68, 72, 74, 72, 68, 74]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.529092747Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"":"downstairs","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.529129978Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"":"downstairs","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.529160379Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"":"downstairs","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.529193669Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"":"downstairs","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"Next flush: 151","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.52922357Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"":"downstairs","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.529252041Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"":"downstairs","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.529281251Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-28T04:52:55.529307762Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"client":"0","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.529337953Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"client":"1","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.529362053Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"client":"2","":"downstairs","session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.529376333Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.529388874Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"e0032ecb-ba2c-445e-8c21-f7edbbda7832 is now active with session: 27329231-dabf-47ae-a995-66e7d2befb72","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.529401684Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.529413744Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491,"session_id":"27329231-dabf-47ae-a995-66e7d2befb72"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:55.529426105Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 54 len 16384 data: 10 9 11 11 02/30 Read block 376 len 4096 03/30 Read block 319 len 20480 04/30 Read block 581 len 4096 05/30 Read block 405 len 24576 06/30 Read block 255 len 4096 07/30 Read block 278 len 24576 08/30 Write block 71 len 24576 data: 12 11 12 13 11 8 09/30 Read block 228 len 12288 10/30 Write block 223 len 16384 data: 7 6 5 3 11/30 Flush 12/30 Write block 477 len 16384 data: 7 8 6 7 13/30 Write block 292 len 8192 data: 6 4 14/30 Read block 246 len 8192 15/30 Read block 23 len 24576 16/30 Write block 267 len 24576 data: 8 8 10 13 12 12 17/30 Read block 339 len 20480 18/30 Read block 164 len 16384 19/30 Read block 391 len 24576 20/30 Write block 238 len 24576 data: 6 5 8 7 6 7 21/30 Write block 150 len 28672 data: 10 10 11 8 8 8 7 22/30 Write block 569 len 40960 data: 8 8 7 7 8 9 9 7 7 7 23/30 Read block 234 len 32768 24/30 Write block 236 len 4096 data: 9 25/30 Read block 408 len 28672 26/30 Read block 112 len 12288 27/30 Read block 457 len 8192 28/30 Write block 50 len 24576 data: 9 8 8 8 11 10 29/30 Write block 231 len 16384 data: 7 8 9 9 30/30 Write block 349 len 16384 data: 8 9 9 9 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 4 Done Done Done false 1012 Acked Write 2 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:52:55.547592189Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1491} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 138 138 138 F F F 1 020-039 70 70 70 138 138 138 F F F 2 040-059 76 76 76 151 151 151 T T T 3 060-079 76 76 76 151 151 151 F F F 4 080-099 72 72 72 141 141 141 F F F 5 100-119 70 70 70 136 136 136 F F F 6 120-139 68 68 68 132 132 132 F F F 7 140-159 72 72 72 140 140 140 T T T 8 160-179 74 74 74 149 149 149 F F F 9 180-199 72 72 72 140 140 140 F F F 10 200-219 68 68 68 131 131 131 F F F 11 220-239 76 76 76 151 151 151 T T T 12 240-259 74 74 74 147 147 147 T T T 13 260-279 73 73 73 142 142 142 T T T 14 280-299 75 75 75 150 150 150 T T T 15 300-319 70 70 70 137 137 137 F F F 16 320-339 70 70 70 138 138 138 F F F 17 340-359 74 74 74 149 149 149 T T T 18 360-379 74 74 74 148 148 148 F F F 19 380-399 75 75 75 150 150 150 F F F 20 400-419 73 73 73 142 142 142 F F F 21 420-439 68 68 68 131 131 131 F F F 22 440-459 70 70 70 137 137 137 F F F 23 460-479 73 73 73 142 142 142 T T T 24 480-499 68 68 68 131 131 131 T T T 25 500-519 68 68 68 131 131 131 F F F 26 520-539 74 74 74 146 146 146 F F F 27 540-559 71 71 71 139 139 139 F F F 28 560-579 60 60 60 113 113 113 T T T 29 580-599 70 70 70 136 136 136 F F F Max gen: 76, Max flush: 151 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 138 138 138 F F F 1 020-039 70 70 70 138 138 138 F F F 2 040-059 77 77 77 152 152 152 F F F 3 060-079 76 76 76 151 151 151 F F F 4 080-099 72 72 72 141 141 141 F F F 5 100-119 70 70 70 136 136 136 F F F 6 120-139 68 68 68 132 132 132 F F F 7 140-159 77 77 77 152 152 152 F F F 8 160-179 74 74 74 149 149 149 F F F 9 180-199 72 72 72 140 140 140 F F F 10 200-219 68 68 68 131 131 131 F F F 11 220-239 77 77 77 152 152 152 F F F 12 240-259 77 77 77 152 152 152 F F F 13 260-279 77 77 77 152 152 152 F F F 14 280-299 77 77 77 152 152 152 F F F 15 300-319 70 70 70 137 137 137 F F F 16 320-339 70 70 70 138 138 138 F F F 17 340-359 77 77 77 152 152 152 F F F 18 360-379 74 74 74 148 148 148 F F F 19 380-399 75 75 75 150 150 150 F F F 20 400-419 73 73 73 142 142 142 F F F 21 420-439 68 68 68 131 131 131 F F F 22 440-459 70 70 70 137 137 137 F F F 23 460-479 77 77 77 152 152 152 F F F 24 480-499 77 77 77 152 152 152 F F F 25 500-519 68 68 68 131 131 131 F F F 26 520-539 74 74 74 146 146 146 F F F 27 540-559 71 71 71 139 139 139 F F F 28 560-579 77 77 77 152 152 152 F F F 29 580-599 70 70 70 136 136 136 F F F Max gen: 77, Max flush: 152 Begin loop 39 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.869126985Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:52:59.912558088Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.912919885Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.945463731Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.946012163Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.946046824Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.946060954Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.946076264Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499} {"msg":"Crucible 1e04af04-67e0-47e4-9d18-abca2c631ad2 has session id: 5826b8a4-28b1-469d-aeee-d54f3ba4601f","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.946536045Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"Upstairs opts: Upstairs UUID: 1e04af04-67e0-47e4-9d18-abca2c631ad2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.946579326Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"Crucible stats registered with UUID: 1e04af04-67e0-47e4-9d18-abca2c631ad2","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.946596306Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.977514397Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.977552667Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499} {"msg":"1e04af04-67e0-47e4-9d18-abca2c631ad2 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.977601939Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.977621809Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"client":"0","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.97765082Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"client":"0","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.97766906Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"client":"1","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.97768607Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"client":"1","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.977703531Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"client":"2","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.977730931Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"client":"2","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.977764952Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"":"io task","client":"2","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.977790003Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"":"io task","client":"0","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.977807843Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"":"io task","client":"1","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"ds_connection connected from Ok(127.0.0.1:65140)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.977839504Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"":"io task","client":"2","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"ds_connection connected from Ok(127.0.0.1:48087)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.978038178Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"":"io task","client":"0","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"ds_connection connected from Ok(127.0.0.1:35180)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.978087199Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"":"io task","client":"1","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.97859313Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"client":"1","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.978642601Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"client":"1","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.978682232Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"client":"2","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.978711072Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"client":"2","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.978739943Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"client":"0","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.978770374Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"client":"0","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.979252334Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"client":"1","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.979281495Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"client":"2","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.979360146Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"client":"0","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.979793996Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"client":"2","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.979822817Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"client":"2","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.979837807Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.979918909Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"client":"1","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.979940709Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"client":"1","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.97995461Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.9799673Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"client":"0","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.97998101Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"client":"0","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"[0]R flush_numbers[0..12]: [138, 138, 152, 151, 141, 136, 132, 152, 149, 140, 131, 152]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.97999525Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"":"downstairs","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"[0]R generation[0..12]: [70, 70, 77, 76, 72, 70, 68, 77, 74, 72, 68, 77]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.980009251Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"":"downstairs","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.980023501Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"":"downstairs","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"[1]R flush_numbers[0..12]: [138, 138, 152, 151, 141, 136, 132, 152, 149, 140, 131, 152]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.980037551Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"":"downstairs","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"[1]R generation[0..12]: [70, 70, 77, 76, 72, 70, 68, 77, 74, 72, 68, 77]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.980051352Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"":"downstairs","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.980065612Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"":"downstairs","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"[2]R flush_numbers[0..12]: [138, 138, 152, 151, 141, 136, 132, 152, 149, 140, 131, 152]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.980079842Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"":"downstairs","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"[2]R generation[0..12]: [70, 70, 77, 76, 72, 70, 68, 77, 74, 72, 68, 77]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.980093833Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"":"downstairs","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.980107873Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"":"downstairs","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.980123153Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"":"downstairs","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.980136654Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"":"downstairs","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"Next flush: 153","v":0,"name":"crucible","level":30,"timeWait for a query_work_queue command to finish before sending IO ":"2024-11-28T04:52:59.980149734Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"":"downstairs","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.980171634Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"":"downstairs","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.980185024Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.980197115Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"client":"0","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.980210425Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"client":"1","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.980223595Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"client":"2","":"downstairs","session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.980237056Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.980249576Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"1e04af04-67e0-47e4-9d18-abca2c631ad2 is now active with session: 5826b8a4-28b1-469d-aeee-d54f3ba4601f","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.980263446Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.980275686Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499,"session_id":"5826b8a4-28b1-469d-aeee-d54f3ba4601f"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:52:59.980288127Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 189 len 28672 data: 10 9 9 8 8 9 10 02/30 Write block 559 len 4096 data: 7 03/30 Write block 229 len 8192 data: 5 5 04/30 Read block 364 len 12288 05/30 Read block 471 len 16384 06/30 Write block 58 len 4096 data: 11 07/30 Write block 242 len 32768 data: 7 8 7 7 9 8 9 9 08/30 Write block 410 len 8192 data: 7 5 09/30 Read block 287 len 32768 10/30 Read block 107 len 4096 11/30 Write block 232 len 12288 data: 9 10 10 12/30 Write block 407 len 4096 data: 5 13/30 Read block 264 len 32768 14/30 Flush 15/30 Write block 458 len 8192 data: 7 9 16/30 Write block 38 len 12288 data: 7 9 9 17/30 Read block 425 len 24576 18/30 Write block 246 len 36864 data: 10 9 10 10 10 9 8 8 8 19/30 Write block 71 len 32768 data: 13 12 13 14 12 9 8 7 20/30 Read block 320 len 12288 21/30 Read block 114 len 20480 22/30 Write block 264 len 12288 data: 5 5 7 23/30 Read block 95 len 28672 24/30 Read block 63 len 20480 25/30 Write block 301 len 24576 data: 7 7 6 4 4 4 26/30 Write block 523 len 32768 data: 7 9 8 11 8 7 7 5 27/30 Write block 41 len 12288 data: 8 8 7 28/30 Read block 364 len 20480 29/30 Read block 487 len 36864 30/30 Write block 429 len 32768 data: 7 6 6 5 5 5 5 5 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 2 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:52:59.99841303Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1499} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 138 138 138 F F F 1 020-039 70 70 70 138 138 138 T T T 2 040-059 78 78 78 153 153 153 T T T 3 060-079 76 76 76 151 151 151 T T T 4 080-099 72 72 72 141 141 141 F F F 5 100-119 70 70 70 136 136 136 F F F 6 120-139 68 68 68 132 132 132 F F F 7 140-159 77 77 77 152 152 152 F F F 8 160-179 74 74 74 149 149 149 F F F 9 180-199 78 78 78 153 153 153 F F F 10 200-219 68 68 68 131 131 131 F F F 11 220-239 78 78 78 153 153 153 F F F 12 240-259 78 78 78 153 153 153 T T T 13 260-279 77 77 77 152 152 152 T T T 14 280-299 77 77 77 152 152 152 F F F 15 300-319 70 70 70 137 137 137 T T T 16 320-339 70 70 70 138 138 138 F F F 17 340-359 77 77 77 152 152 152 F F F 18 360-379 74 74 74 148 148 148 F F F 19 380-399 75 75 75 150 150 150 F F F 20 400-419 78 78 78 153 153 153 F F F 21 420-439 68 68 68 131 131 131 T T T 22 440-459 70 70 70 137 137 137 T T T 23 460-479 77 77 77 152 152 152 F F F 24 480-499 77 77 77 152 152 152 F F F 25 500-519 68 68 68 131 131 131 F F F 26 520-539 74 74 74 146 146 146 T T T 27 540-559 78 78 78 153 153 153 F F F 28 560-579 77 77 77 152 152 152 F F F 29 580-599 70 70 70 136 136 136 F F F Max gen: 78, Max flush: 153 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 138 138 138 F F F 1 020-039 79 79 79 154 154 154 F F F 2 040-059 79 79 79 154 154 154 F F F 3 060-079 79 79 79 154 154 154 F F F 4 080-099 72 72 72 141 141 141 F F F 5 100-119 70 70 70 136 136 136 F F F 6 120-139 68 68 68 132 132 132 F F F 7 140-159 77 77 77 152 152 152 F F F 8 160-179 74 74 74 149 149 149 F F F 9 180-199 78 78 78 153 153 153 F F F 10 200-219 68 68 68 131 131 131 F F F 11 220-239 78 78 78 153 153 153 F F F 12 240-259 79 79 79 154 154 154 F F F 13 260-279 79 79 79 154 154 154 F F F 14 280-299 77 77 77 152 152 152 F F F 15 300-319 79 79 79 154 154 154 F F F 16 320-339 70 70 70 138 138 138 F F F 17 340-359 77 77 77 152 152 152 F F F 18 360-379 74 74 74 148 148 148 F F F 19 380-399 75 75 75 150 150 150 F F F 20 400-419 78 78 78 153 153 153 F F F 21 420-439 79 79 79 154 154 154 F F F 22 440-459 79 79 79 154 154 154 F F F 23 460-479 77 77 77 152 152 152 F F F 24 480-499 77 77 77 152 152 152 F F F 25 500-519 68 68 68 131 131 131 F F F 26 520-539 79 79 79 154 154 154 F F F 27 540-559 78 78 78 153 153 153 F F F 28 560-579 77 77 77 152 152 152 F F F 29 580-599 70 70 70 136 136 136 F F F Max gen: 79, Max flush: 154 Begin loop 40 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.313449696Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.360017127Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.36065689Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.360693101Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.360707491Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.360722202Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507} {"msg":"Crucible b0c7c5be-d8fc-4100-814a-3ea466729a5d has session id: 44c4bbbc-df73-46fb-9c86-eb81e1a761b0","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.361144981Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"Upstairs opts: Upstairs UUID: b0c7c5be-d8fc-4100-814a-3ea466729a5d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.361186962Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"Crucible stats registered with UUID: b0c7c5be-d8fc-4100-814a-3ea466729a5d","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.361204012Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.392574113Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.392613754Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507} {"msg":"b0c7c5be-d8fc-4100-814a-3ea466729a5d active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.392643754Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.392665795Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"client":"0","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.392711646Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"client":"0","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.392748666Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"client":"1","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.392785007Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"client":"1","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.392822488Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"client":"2","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.392851369Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"client":"2","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.39287962Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"":"io task","client":"2","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.392934941Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"":"io task","client":"0","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.393175346Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"":"io task","client":"1","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:42564)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.393211067Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"":"io task","client":"2","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:58449)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.393242907Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"":"io task","client":"1","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:59679)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.393278228Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"":"io task","client":"0","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.393742748Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"client":"0","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.393792519Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"client":"0","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.39383071Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"client":"2","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.393861701Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"client":"2","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.393889971Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"client":"1","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.393905952Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"client":"1","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.394329481Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"client":"0","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.394401582Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"client":"1","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.394621757Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"client":"2","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.394931034Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"client":"0","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.394960874Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"client":"0","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.394975645Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.395065107Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"client":"1","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.395084397Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"client":"1","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.395102078Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.395129168Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"client":"2","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.395143458Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"client":"2","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"[0]R flush_numbers[0..12]: [138, 154, 154, 154, 141, 136, 132, 152, 149, 153, 131, 153]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.395170029Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"":"downstairs","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"[0]R generation[0..12]: [70, 79, 79, 79, 72, 70, 68, 77, 74, 78, 68, 78]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.395184419Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"":"downstairs","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.39519854Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"":"downstairs","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"[1]R flush_numbers[0..12]: [138, 154, 154, 154, 141, 136, 132, 152, 149, 153, 131, 153]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.39521271Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"":"downstairs","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"[1]R generation[0..12]: [70, 79, 79, 79, 72, 70, 68, 77, 74, 78, 68, 78]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.39522658Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"":"downstairs","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.395240721Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"":"downstairs","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"[2]R flush_numbers[0..12]: [138, 154, 154, 154, 141, 136, 132, 152, 149, 153, 131, 153]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.395254881Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"":"downstairs","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"[2]R generation[0..12]: [70, 79, 79, 79, 72, 70, 68, 77, 74, 78, 68, 78]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.395268581Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"":"downstairs","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.395282732Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"":"downstairs","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.395295922Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"":"downstairs","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.395309302Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"":"downstairs","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"Next flush: 155","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.395324373Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"":"downstairs","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.395337793Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"":"downstairs","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"No downstairs reconciliation required","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2024-11-28T04:53:04.395359903Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.395383654Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"client":"0","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.395410184Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"client":"1","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.395436315Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"client":"2","":"downstairs","session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.395455645Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.395468605Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"b0c7c5be-d8fc-4100-814a-3ea466729a5d is now active with session: 44c4bbbc-df73-46fb-9c86-eb81e1a761b0","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.395481616Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.395498286Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507,"session_id":"44c4bbbc-df73-46fb-9c86-eb81e1a761b0"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:04.395510897Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 258 len 24576 data: 8 7 6 6 5 3 02/30 Read block 91 len 36864 03/30 Write block 252 len 8192 data: 9 9 04/30 Read block 74 len 40960 05/30 Read block 96 len 12288 06/30 Write block 191 len 20480 data: 10 9 9 10 11 07/30 Write block 496 len 36864 data: 7 7 8 8 12 9 7 9 7 08/30 Read block 524 len 32768 09/30 Read block 294 len 36864 10/30 Read block 530 len 40960 11/30 Read block 493 len 32768 12/30 Read block 234 len 12288 13/30 Write block 550 len 24576 data: 8 7 8 8 8 8 14/30 Read block 294 len 40960 15/30 Read block 402 len 28672 16/30 Flush 17/30 Write block 401 len 20480 data: 8 7 6 7 6 18/30 Write block 27 len 36864 data: 7 8 7 3 4 4 4 4 5 19/30 Write block 153 len 12288 data: 9 9 9 20/30 Read block 56 len 4096 21/30 Flush 22/30 Write block 141 len 40960 data: 5 5 4 4 5 6 8 9 11 11 23/30 Write block 247 len 36864 data: 10 11 11 11 10 10 10 9 8 24/30 Write block 361 len 28672 data: 8 7 7 5 5 6 6 25/30 Write block 581 len 8192 data: 4 4 26/30 Read block 374 len 28672 27/30 Write block 130 len 4096 data: 4 28/30 Read block 433 len 36864 29/30 Read block 258 len 32768 30/30 Read block 178 len 24576 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 10 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 9 8 8 25 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:53:04.415369387Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1507} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 138 138 138 F F F 1 020-039 80 80 80 156 156 156 F F F 2 040-059 79 79 79 154 154 154 F F F 3 060-079 79 79 79 154 154 154 F F F 4 080-099 72 72 72 141 141 141 F F F 5 100-119 70 70 70 136 136 136 F F F 6 120-139 68 68 68 132 132 132 T T T 7 140-159 80 80 80 156 156 156 T T T 8 160-179 74 74 74 149 149 149 F F F 9 180-199 80 80 80 155 155 155 F F F 10 200-219 68 68 68 131 131 131 F F F 11 220-239 78 78 78 153 153 153 F F F 12 240-259 80 80 80 155 155 155 T T T 13 260-279 80 80 80 155 155 155 F F F 14 280-299 77 77 77 152 152 152 F F F 15 300-319 79 79 79 154 154 154 F F F 16 320-339 70 70 70 138 138 138 F F F 17 340-359 77 77 77 152 152 152 F F F 18 360-379 74 74 74 148 148 148 T T T 19 380-399 75 75 75 150 150 150 F F F 20 400-419 80 80 80 156 156 156 F F F 21 420-439 79 79 79 154 154 154 F F F 22 440-459 79 79 79 154 154 154 F F F 23 460-479 77 77 77 152 152 152 F F F 24 480-499 80 80 80 155 155 155 F F F 25 500-519 80 80 80 155 155 155 F F F 26 520-539 79 79 79 154 154 154 F F F 27 540-559 80 80 80 155 155 155 F F F 28 560-579 77 77 77 152 152 152 F F F 29 580-599 70 70 70 136 136 136 T T T Max gen: 80, Max flush: 156 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 138 138 138 F F F 1 020-039 80 80 80 156 156 156 F F F 2 040-059 79 79 79 154 154 154 F F F 3 060-079 79 79 79 154 154 154 F F F 4 080-099 72 72 72 141 141 141 F F F 5 100-119 70 70 70 136 136 136 F F F 6 120-139 81 81 81 157 157 157 F F F 7 140-159 81 81 81 157 157 157 F F F 8 160-179 74 74 74 149 149 149 F F F 9 180-199 80 80 80 155 155 155 F F F 10 200-219 68 68 68 131 131 131 F F F 11 220-239 78 78 78 153 153 153 F F F 12 240-259 81 81 81 157 157 157 F F F 13 260-279 80 80 80 155 155 155 F F F 14 280-299 77 77 77 152 152 152 F F F 15 300-319 79 79 79 154 154 154 F F F 16 320-339 70 70 70 138 138 138 F F F 17 340-359 77 77 77 152 152 152 F F F 18 360-379 81 81 81 157 157 157 F F F 19 380-399 75 75 75 150 150 150 F F F 20 400-419 80 80 80 156 156 156 F F F 21 420-439 79 79 79 154 154 154 F F F 22 440-459 79 79 79 154 154 154 F F F 23 460-479 77 77 77 152 152 152 F F F 24 480-499 80 80 80 155 155 155 F F F 25 500-519 80 80 80 155 155 155 F F F 26 520-539 79 79 79 154 154 154 F F F 27 540-559 80 80 80 155 155 155 F F F 28 560-579 77 77 77 152 152 152 F F F 29 580-599 81 81 81 157 157 157 F F F Max gen: 81, Max flush: 157 Begin loop 41 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.681195756Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.727153513Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.727726925Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.727762086Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.727776636Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.727791057Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515} {"msg":"Crucible d03bfa2c-037c-483f-afa0-7b903cc758fe has session id: 428cf387-2048-4ca1-babc-60f20a726f5d","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.728213556Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"Upstairs opts: Upstairs UUID: d03bfa2c-037c-483f-afa0-7b903cc758fe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.728250767Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"Crucible stats registered with UUID: d03bfa2c-037c-483f-afa0-7b903cc758fe","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.728268457Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.758995254Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.759028704Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515} {"msg":"d03bfa2c-037c-483f-afa0-7b903cc758fe active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.759056495Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.759073015Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"client":"0","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.759101236Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"client":"0","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.759119416Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"client":"1","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.759136847Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"client":"1","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.759153697Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"client":"2","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.759170747Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"client":"2","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.759187548Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.759204818Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"":"io task","client":"0","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.759221959Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"":"io task","client":"1","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"ds_connection connected from Ok(127.0.0.1:63830)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.759459294Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"ds_connection connected from Ok(127.0.0.1:58214)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.759512405Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"":"io task","client":"1","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"ds_connection connected from Ok(127.0.0.1:33748)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.759541635Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"":"io task","client":"0","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.76021334Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"client":"0","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.760243521Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"client":"0","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.760264551Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"client":"1","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.760358163Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"client":"1","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.760440175Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"client":"2","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.760483866Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"client":"2","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.760886415Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"client":"0","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761042808Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"client":"2","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761206931Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"client":"1","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761513838Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"client":"0","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761539959Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"client":"0","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761568759Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761611Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"client":"2","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761629911Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"client":"2","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761644641Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761682482Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"client":"1","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761699952Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"client":"1","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"[0]R flush_numbers[0..12]: [138, 156, 154, 154, 141, 136, 157, 157, 149, 155, 131, 153]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761717573Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"":"downstairs","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"[0]R generation[0..12]: [70, 80, 79, 79, 72, 70, 81, 81, 74, 80, 68, 78]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761732893Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"":"downstairs","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761747303Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"":"downstairs","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"[1]R flush_numbers[0..12]: [138, 156, 154, 154, 141, 136, 157, 157, 149, 155, 131, 153]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761761423Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"":"downstairs","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"[1]R generation[0..12]: [70, 80, 79, 79, 72, 70, 81, 81, 74, 80, 68, 78]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761775184Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"":"downstairs","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761789214Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"":"downstairs","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"[2]R flush_numbers[0..12]: [138, 156, 154, 154, 141, 136, 157, 157, 149, 155, 131, 153]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761803244Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"":"downstairs","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"[2]R generation[0..12]: [70, 80, 79, 79, 72, 70, 81, 81, 74, 80, 68, 78]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761817045Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"":"downstairs","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761830995Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"":"downstairs","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761844305Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"":"downstairs","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761857696Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"":"downstairs","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"Next flush: 158","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761870886Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"":"downstairs","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761884016Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":Wait for a query_work_queue command to finish before sending IO 1515,"":"downstairs","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761906257Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761921197Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"client":"0","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761934947Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"client":"1","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761948548Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"client":"2","":"downstairs","session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761962508Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761977108Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"d03bfa2c-037c-483f-afa0-7b903cc758fe is now active with session: 428cf387-2048-4ca1-babc-60f20a726f5d","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.761990348Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.762002839Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515,"session_id":"428cf387-2048-4ca1-babc-60f20a726f5d"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:08.762015329Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 572 len 20480 data: 8 9 10 10 8 02/30 Write block 271 len 8192 data: 13 13 03/30 Write block 524 len 8192 data: 10 9 04/30 Read block 267 len 40960 05/30 Write block 135 len 40960 data: 6 5 4 5 5 5 6 6 5 5 06/30 Read block 370 len 36864 07/30 Write block 403 len 36864 data: 7 8 7 7 6 7 6 8 6 08/30 Read block 264 len 28672 09/30 Read block 393 len 24576 10/30 Write block 542 len 20480 data: 4 4 4 7 7 11/30 Write block 181 len 28672 data: 3 5 8 10 10 12 11 12/30 Read block 477 len 20480 13/30 Write block 383 len 12288 data: 4 4 4 14/30 Write block 269 len 8192 data: 11 14 15/30 Read block 421 len 32768 16/30 Write block 404 len 40960 data: 9 8 8 7 8 7 9 7 6 7 17/30 Read block 110 len 32768 18/30 Flush 19/30 Write block 476 len 40960 data: 9 8 9 7 8 7 7 5 4 4 20/30 Read block 568 len 28672 21/30 Write block 234 len 12288 data: 11 9 10 22/30 Read block 579 len 12288 23/30 Write block 205 len 8192 data: 5 5 24/30 Read block 345 len 8192 25/30 Read block 114 len 40960 26/30 Write block 527 len 16384 data: 9 8 8 6 27/30 Read block 460 len 24576 28/30 Read block 301 len 32768 29/30 Read block 23 len 36864 30/30 Write block 215 len 36864 data: 7 7 3 4 5 5 7 7 8 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 10 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:53:08.782402441Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1515} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 138 138 138 F F F 1 020-039 80 80 80 156 156 156 F F F 2 040-059 79 79 79 154 154 154 F F F 3 060-079 79 79 79 154 154 154 F F F 4 080-099 72 72 72 141 141 141 F F F 5 100-119 70 70 70 136 136 136 F F F 6 120-139 82 82 82 158 158 158 F F F 7 140-159 82 82 82 158 158 158 F F F 8 160-179 74 74 74 149 149 149 F F F 9 180-199 82 82 82 158 158 158 F F F 10 200-219 68 68 68 131 131 131 T T T 11 220-239 78 78 78 153 153 153 T T T 12 240-259 81 81 81 157 157 157 F F F 13 260-279 82 82 82 158 158 158 F F F 14 280-299 77 77 77 152 152 152 F F F 15 300-319 79 79 79 154 154 154 F F F 16 320-339 70 70 70 138 138 138 F F F 17 340-359 77 77 77 152 152 152 F F F 18 360-379 81 81 81 157 157 157 F F F 19 380-399 82 82 82 158 158 158 F F F 20 400-419 82 82 82 158 158 158 F F F 21 420-439 79 79 79 154 154 154 F F F 22 440-459 79 79 79 154 154 154 F F F 23 460-479 77 77 77 152 152 152 T T T 24 480-499 80 80 80 155 155 155 T T T 25 500-519 80 80 80 155 155 155 F F F 26 520-539 82 82 82 158 158 158 T T T 27 540-559 82 82 82 158 158 158 F F F 28 560-579 82 82 82 158 158 158 F F F 29 580-599 81 81 81 157 157 157 F F F Max gen: 82, Max flush: 158 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 138 138 138 F F F 1 020-039 80 80 80 156 156 156 F F F 2 040-059 79 79 79 154 154 154 F F F 3 060-079 79 79 79 154 154 154 F F F 4 080-099 72 72 72 141 141 141 F F F 5 100-119 70 70 70 136 136 136 F F F 6 120-139 82 82 82 158 158 158 F F F 7 140-159 82 82 82 158 158 158 F F F 8 160-179 74 74 74 149 149 149 F F F 9 180-199 82 82 82 158 158 158 F F F 10 200-219 83 83 83 159 159 159 F F F 11 220-239 83 83 83 159 159 159 F F F 12 240-259 81 81 81 157 157 157 F F F 13 260-279 82 82 82 158 158 158 F F F 14 280-299 77 77 77 152 152 152 F F F 15 300-319 79 79 79 154 154 154 F F F 16 320-339 70 70 70 138 138 138 F F F 17 340-359 77 77 77 152 152 152 F F F 18 360-379 81 81 81 157 157 157 F F F 19 380-399 82 82 82 158 158 158 F F F 20 400-419 82 82 82 158 158 158 F F F 21 420-439 79 79 79 154 154 154 F F F 22 440-459 79 79 79 154 154 154 F F F 23 460-479 83 83 83 159 159 159 F F F 24 480-499 83 83 83 159 159 159 F F F 25 500-519 80 80 80 155 155 155 F F F 26 520-539 83 83 83 159 159 159 F F F 27 540-559 82 82 82 158 158 158 F F F 28 560-579 82 82 82 158 158 158 F F F 29 580-599 81 81 81 157 157 157 F F F Max gen: 83, Max flush: 159 Begin loop 42 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.053223768Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.096787413Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.097357465Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.097394076Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.097408676Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.097423107Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523} {"msg":"Crucible 22861e90-360b-47a9-b741-8ef7855f7c4e has session id: f9c4be6a-ce03-4129-b8d1-676eff36f728","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.097880607Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"Upstairs opts: Upstairs UUID: 22861e90-360b-47a9-b741-8ef7855f7c4e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.097926108Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"Crucible stats registered with UUID: 22861e90-360b-47a9-b741-8ef7855f7c4e","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.097954948Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.128537682Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.128566312Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523} {"msg":"22861e90-360b-47a9-b741-8ef7855f7c4e active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.128634124Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.128670615Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"client":"0","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.128703365Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"client":"0","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.128722036Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"client":"1","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.128740206Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"client":"1","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.128770587Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"client":"2","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.128803108Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"client":"2","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.128834468Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.128869859Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.12890671Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"ds_connection connected from Ok(127.0.0.1:36791)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.128940511Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"ds_connection connected from Ok(127.0.0.1:62444)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.128972731Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"ds_connection connected from Ok(127.0.0.1:47781)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.129006932Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.129539824Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"client":"1","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.129590465Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"client":"1","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.129620785Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"client":"0","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.129671426Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"client":"0","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.129696507Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"client":"2","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.129711227Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"client":"2","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.130234148Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"client":"0","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.130264789Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"client":"2","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.13029307Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"client":"1","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.130828201Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"client":"0","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.130868512Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"client":"0","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.130886903Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.130900063Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"client":"2","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.130913763Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"client":"2","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.130927224Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.130939524Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"client":"1","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.130952834Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"client":"1","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"[0]R flush_numbers[0..12]: [138, 156, 154, 154, 141, 136, 158, 158, 149, 158, 159, 159]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.130966664Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"":"downstairs","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"[0]R generation[0..12]: [70, 80, 79, 79, 72, 70, 82, 82, 74, 82, 83, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.130980645Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"":"downstairs","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.130997815Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"":"downstairs","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"[1]R flush_numbers[0..12]: [138, 156, 154, 154, 141, 136, 158, 158, 149, 158, 159, 159]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.131012145Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"":"downstairs","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"[1]R generation[0..12]: [70, 80, 79, 79, 72, 70, 82, 82, 74, 82, 83, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.131025976Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"":"downstairs","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.131039856Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"":"downstairs","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"[2]R flush_numbers[0..12]: [138, 156, 154, 154, 141, 136, 158, 158, 149, 158, 159, 159]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.131053616Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid"Wait for a query_work_queue command to finish before sending IO :1523,"":"downstairs","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"[2]R generation[0..12]: [70, 80, 79, 79, 72, 70, 82, 82, 74, 82, 83, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.131076267Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"":"downstairs","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.131090437Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"":"downstairs","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.131103467Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"":"downstairs","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.131116768Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"":"downstairs","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"Next flush: 160","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.131129808Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"":"downstairs","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.131142988Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"":"downstairs","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.131156239Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.131168349Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"client":"0","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.131181609Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"client":"1","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.13119674Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"client":"2","":"downstairs","session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.13121033Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.13122273Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"22861e90-360b-47a9-b741-8ef7855f7c4e is now active with session: f9c4be6a-ce03-4129-b8d1-676eff36f728","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.13123545Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.131247681Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523,"session_id":"f9c4be6a-ce03-4129-b8d1-676eff36f728"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:13.131260131Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 135 len 4096 data: 7 02/30 Write block 160 len 36864 data: 5 5 5 5 5 4 5 5 7 03/30 Write block 37 len 16384 data: 6 8 10 10 04/30 Write block 337 len 8192 data: 9 11 05/30 Write block 484 len 12288 data: 5 5 5 06/30 Read block 42 len 4096 07/30 Read block 137 len 28672 08/30 Read block 302 len 4096 09/30 Read block 94 len 40960 10/30 Read block 205 len 40960 11/30 Read block 196 len 36864 12/30 Flush 13/30 Write block 117 len 24576 data: 6 5 5 4 6 6 14/30 Read block 587 len 12288 15/30 Read block 165 len 4096 16/30 Write block 429 len 32768 data: 8 7 7 6 6 6 6 6 17/30 Read block 105 len 16384 18/30 Write block 132 len 8192 data: 5 5 19/30 Write block 451 len 16384 data: 8 7 8 10 20/30 Read block 82 len 20480 21/30 Flush 22/30 Write block 552 len 20480 data: 9 9 9 9 10 23/30 Write block 72 len 40960 data: 13 14 15 13 10 9 8 7 4 5 24/30 Write block 558 len 8192 data: 7 8 25/30 Read block 101 len 20480 26/30 Flush 27/30 Write block 452 len 40960 data: 8 9 11 10 11 8 8 10 8 9 28/30 Read block 406 len 24576 29/30 Read block 402 len 16384 30/30 Read block 249 len 20480 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 10 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 5 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:53:13.15012202Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1523} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 138 138 138 F F F 1 020-039 84 84 84 160 160 160 F F F 2 040-059 84 84 84 160 160 160 F F F 3 060-079 84 84 84 162 162 162 F F F 4 080-099 84 84 84 162 162 162 F F F 5 100-119 84 84 84 161 161 161 F F F 6 120-139 84 84 84 161 161 161 F F F 7 140-159 82 82 82 158 158 158 F F F 8 160-179 84 84 84 160 160 160 F F F 9 180-199 82 82 82 158 158 158 F F F 10 200-219 83 83 83 159 159 159 F F F 11 220-239 83 83 83 159 159 159 F F F 12 240-259 81 81 81 157 157 157 F F F 13 260-279 82 82 82 158 158 158 F F F 14 280-299 77 77 77 152 152 152 F F F 15 300-319 79 79 79 154 154 154 F F F 16 320-339 84 84 84 160 160 160 F F F 17 340-359 77 77 77 152 152 152 F F F 18 360-379 81 81 81 157 157 157 F F F 19 380-399 82 82 82 158 158 158 F F F 20 400-419 82 82 82 158 158 158 F F F 21 420-439 84 84 84 161 161 161 F F F 22 440-459 84 84 84 161 161 161 T T T 23 460-479 83 83 83 159 159 159 T T T 24 480-499 84 84 84 160 160 160 F F F 25 500-519 80 80 80 155 155 155 F F F 26 520-539 83 83 83 159 159 159 F F F 27 540-559 84 84 84 162 162 162 F F F 28 560-579 82 82 82 158 158 158 F F F 29 580-599 81 81 81 157 157 157 F F F Max gen: 84, Max flush: 162 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 138 138 138 F F F 1 020-039 84 84 84 160 160 160 F F F 2 040-059 84 84 84 160 160 160 F F F 3 060-079 84 84 84 162 162 162 F F F 4 080-099 84 84 84 162 162 162 F F F 5 100-119 84 84 84 161 161 161 F F F 6 120-139 84 84 84 161 161 161 F F F 7 140-159 82 82 82 158 158 158 F F F 8 160-179 84 84 84 160 160 160 F F F 9 180-199 82 82 82 158 158 158 F F F 10 200-219 83 83 83 159 159 159 F F F 11 220-239 83 83 83 159 159 159 F F F 12 240-259 81 81 81 157 157 157 F F F 13 260-279 82 82 82 158 158 158 F F F 14 280-299 77 77 77 152 152 152 F F F 15 300-319 79 79 79 154 154 154 F F F 16 320-339 84 84 84 160 160 160 F F F 17 340-359 77 77 77 152 152 152 F F F 18 360-379 81 81 81 157 157 157 F F F 19 380-399 82 82 82 158 158 158 F F F 20 400-419 82 82 82 158 158 158 F F F 21 420-439 84 84 84 161 161 161 F F F 22 440-459 85 85 85 163 163 163 F F F 23 460-479 85 85 85 163 163 163 F F F 24 480-499 84 84 84 160 160 160 F F F 25 500-519 80 80 80 155 155 155 F F F 26 520-539 83 83 83 159 159 159 F F F 27 540-559 84 84 84 162 162 162 F F F 28 560-579 82 82 82 158 158 158 F F F 29 580-599 81 81 81 157 157 157 F F F Max gen: 85, Max flush: 163 Begin loop 43 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.396996847Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:53:17.43951036Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.439899128Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.47131794Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.471867632Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.471903632Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.471917973Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.471932623Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531} {"msg":"Crucible ee9e0a73-9e75-420e-b817-877bca2851c7 has session id: 2d0a28ca-5235-4c78-97be-2711fe177ce9","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.472415744Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"Upstairs opts: Upstairs UUID: ee9e0a73-9e75-420e-b817-877bca2851c7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.472464365Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"Crucible stats registered with UUID: ee9e0a73-9e75-420e-b817-877bca2851c7","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.472489655Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.503399846Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.503431296Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531} {"msg":"ee9e0a73-9e75-420e-b817-877bca2851c7 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.503461377Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.503478397Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"client":"0","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.503507258Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"client":"0","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.503525029Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"client":"1","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.503545389Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"client":"1","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.503562339Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"client":"2","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.5035801Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"client":"2","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.50359737Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"":"io task","client":"2","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.503614871Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"":"io task","client":"0","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.503647261Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"":"io task","client":"1","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"ds_connection connected from Ok(127.0.0.1:48021)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.503682892Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"":"io task","client":"2","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"ds_connection connected from Ok(127.0.0.1:53371)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.503718023Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"":"io task","client":"0","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"ds_connection connected from Ok(127.0.0.1:57533)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.503951518Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"":"io task","client":"1","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.504476389Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"client":"1","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.504549551Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"client":"1","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.504588601Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"client":"2","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.504619922Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"client":"2","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.504636953Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"client":"0","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.504651243Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"client":"0","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.505081112Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"client":"1","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.505180385Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"client":"0","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.505310387Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"client":"2","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.505666485Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"client":"1","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.505707696Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"client":"1","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.505737806Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.505802548Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"client":"0","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.505833679Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"client":"0","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.505851269Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.505865549Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"client":"2","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.50588069Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"client":"2","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"[0]R flush_numbers[0..12]: [138, 160, 160, 162, 162, 161, 161, 158, 160, 158, 159, 159]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.505896Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"":"downstairs","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"[0]R generation[0..12]: [70, 84, 84, 84, 84, 84, 84, 82, 84, 82, 83, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.50591078Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"":"downstairs","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.505926131Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"":"downstairs","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"[1]R flush_numbers[0..12]: [138, 160, 160, 162, 162, 161, 161, 158, 160, 158, 159, 159]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.505941221Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"":"downstairs","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"[1]R generation[0..12]: [70, 84, 84, 84, 84, 84, 84, 82, 84, 82, 83, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.505959161Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"":"downstairs","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.505975112Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"":"downstairs","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"[2]R flush_numbers[0..12]: [138, 160, 160, 162, 162, 161, 161, 158, 160, 158, 159, 159]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.505990412Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"":"downstairs","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"[2]R generation[0..12]: [70, 84, 84, 84, 84, 84, 84, 82, 84, 82, 83, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.506005452Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"":"downstairs","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.506020843Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"":"downstairs","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.506035003Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"":"downstairs","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.506049783Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"":"downstairs","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"Next flush: 164","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.506064073Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"":"downstairs","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.506078404Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"":"downstairs","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"No downstairs reconciliation required","Wait for a query_work_queue command to finish before sending IO v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.506102354Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.506115915Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"client":"0","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.506130195Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"client":"1","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.506144815Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"client":"2","":"downstairs","session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.506159566Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.506172506Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"ee9e0a73-9e75-420e-b817-877bca2851c7 is now active with session: 2d0a28ca-5235-4c78-97be-2711fe177ce9","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.506188526Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.506202347Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531,"session_id":"2d0a28ca-5235-4c78-97be-2711fe177ce9"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:17.506215987Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 0 len 8192 data: 2 3 02/30 Flush 03/30 Write block 210 len 40960 data: 10 10 9 8 7 8 8 4 5 6 04/30 Flush 05/30 Flush 06/30 Write block 256 len 12288 data: 7 8 9 07/30 Write block 373 len 40960 data: 5 5 6 6 7 6 6 5 5 5 08/30 Read block 119 len 8192 09/30 Write block 591 len 36864 data: 6 6 4 3 3 3 4 3 3 10/30 Read block 166 len 16384 11/30 Flush 12/30 Write block 202 len 24576 data: 8 7 5 6 6 5 13/30 Write block 195 len 40960 data: 12 8 10 10 8 8 7 9 8 6 14/30 Flush 15/30 Write block 81 len 8192 data: 6 4 16/30 Read block 12 len 20480 17/30 Read block 322 len 8192 18/30 Read block 28 len 32768 19/30 Write block 304 len 28672 data: 5 5 5 5 5 3 3 20/30 Write block 7 len 40960 data: 5 6 5 3 5 5 7 6 8 8 21/30 Write block 433 len 16384 data: 7 7 7 7 22/30 Read block 45 len 4096 23/30 Read block 476 len 8192 24/30 Read block 261 len 4096 25/30 Read block 458 len 8192 26/30 Read block 365 len 12288 27/30 Read block 467 len 20480 28/30 Read block 370 len 4096 29/30 Read block 589 len 16384 30/30 Read block 51 len 40960 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 2 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 10 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 15 16 16 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:53:17.525628598Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1531} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 164 164 164 T T T 1 020-039 84 84 84 160 160 160 F F F 2 040-059 84 84 84 160 160 160 F F F 3 060-079 84 84 84 162 162 162 F F F 4 080-099 84 84 84 162 162 162 T T T 5 100-119 84 84 84 161 161 161 F F F 6 120-139 84 84 84 161 161 161 F F F 7 140-159 82 82 82 158 158 158 F F F 8 160-179 84 84 84 160 160 160 F F F 9 180-199 86 86 86 168 168 168 F F F 10 200-219 86 86 86 168 168 168 F F F 11 220-239 83 83 83 159 159 159 F F F 12 240-259 86 86 86 167 167 167 F F F 13 260-279 82 82 82 158 158 158 F F F 14 280-299 77 77 77 152 152 152 F F F 15 300-319 79 79 79 154 154 154 T T T 16 320-339 84 84 84 160 160 160 F F F 17 340-359 77 77 77 152 152 152 F F F 18 360-379 86 86 86 167 167 167 F F F 19 380-399 86 86 86 167 167 167 F F F 20 400-419 82 82 82 158 158 158 F F F 21 420-439 84 84 84 161 161 161 T T T 22 440-459 85 85 85 163 163 163 F F F 23 460-479 85 85 85 163 163 163 F F F 24 480-499 84 84 84 160 160 160 F F F 25 500-519 80 80 80 155 155 155 F F F 26 520-539 83 83 83 159 159 159 F F F 27 540-559 84 84 84 162 162 162 F F F 28 560-579 82 82 82 158 158 158 F F F 29 580-599 86 86 86 167 167 167 F F F Max gen: 86, Max flush: 168 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 169 169 169 F F F 1 020-039 84 84 84 160 160 160 F F F 2 040-059 84 84 84 160 160 160 F F F 3 060-079 84 84 84 162 162 162 F F F 4 080-099 87 87 87 169 169 169 F F F 5 100-119 84 84 84 161 161 161 F F F 6 120-139 84 84 84 161 161 161 F F F 7 140-159 82 82 82 158 158 158 F F F 8 160-179 84 84 84 160 160 160 F F F 9 180-199 86 86 86 168 168 168 F F F 10 200-219 86 86 86 168 168 168 F F F 11 220-239 83 83 83 159 159 159 F F F 12 240-259 86 86 86 167 167 167 F F F 13 260-279 82 82 82 158 158 158 F F F 14 280-299 77 77 77 152 152 152 F F F 15 300-319 87 87 87 169 169 169 F F F 16 320-339 84 84 84 160 160 160 F F F 17 340-359 77 77 77 152 152 152 F F F 18 360-379 86 86 86 167 167 167 F F F 19 380-399 86 86 86 167 167 167 F F F 20 400-419 82 82 82 158 158 158 F F F 21 420-439 87 87 87 169 169 169 F F F 22 440-459 85 85 85 163 163 163 F F F 23 460-479 85 85 85 163 163 163 F F F 24 480-499 84 84 84 160 160 160 F F F 25 500-519 80 80 80 155 155 155 F F F 26 520-539 83 83 83 159 159 159 F F F 27 540-559 84 84 84 162 162 162 F F F 28 560-579 82 82 82 158 158 158 F F F 29 580-599 86 86 86 167 167 167 F F F Max gen: 87, Max flush: 169 Begin loop 44 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.816342006Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:53:21.858799267Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.859153275Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.89073206Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.891328843Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.891363244Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.891377554Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.891392304Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539} {"msg":"Crucible a07eadd3-5868-4a6a-a0cb-865b60519a0f has session id: fd31af8d-5057-4665-9d24-a8ea0297d2f1","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.891778673Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"Upstairs opts: Upstairs UUID: a07eadd3-5868-4a6a-a0cb-865b60519a0f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.891823194Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"Crucible stats registered with UUID: a07eadd3-5868-4a6a-a0cb-865b60519a0f","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.891844534Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.923191084Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.923226615Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539} {"msg":"a07eadd3-5868-4a6a-a0cb-865b60519a0f active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.923261716Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.923290326Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"client":"0","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.923337927Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"client":"0","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.923375498Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"client":"1","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.923409419Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"client":"1","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.92343961Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"client":"2","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.92346914Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"client":"2","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.923498041Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"":"io task","client":"2","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.923528701Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"":"io task","client":"0","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.923558532Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"":"io task","client":"1","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"ds_connection connected from Ok(127.0.0.1:44895)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.923588683Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"":"io task","client":"2","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"ds_connection connected from Ok(127.0.0.1:61877)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.923618853Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"":"io task","client":"1","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"ds_connection connected from Ok(127.0.0.1:33370)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.923746576Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"":"io task","client":"0","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.924210786Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"client":"1","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.924249387Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"client":"1","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.924281048Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"client":"2","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.92436169Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"client":"2","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.92440083Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"client":"0","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.924434651Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"client":"0","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.92483283Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"client":"2","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.924868501Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"client":"1","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.924940972Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"client":"0","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.925447553Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"client":"1","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.925476044Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"client":"1","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.925490564Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.925503064Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"client":"2","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.925516635Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"client":"2","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.925540495Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.925554895Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"client":"0","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.925568476Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"client":"0","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"[0]R flush_numbers[0..12]: [169, 160, 160, 162, 169, 161, 161, 158, 160, 168, 168, 159]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.925582256Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"":"downstairs","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"[0]R generation[0..12]: [87, 84, 84, 84, 87, 84, 84, 82, 84, 86, 86, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.925596046Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"":"downstairs","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.925610326Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"":"downstairs","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"[1]R flush_numbers[0..12]: [169, 160, 160, 162, 169, 161, 161, 158, 160, 168, 168, 159]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.925624047Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"":"downstairs","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"[1]R generation[0..12]: [87, 84, 84, 84, 87, 84, 84, 82, 84, 86, 86, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.925640917Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"":"downstairs","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.925655407Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"":"downstairs","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"[2]R flush_numbers[0..12]: [169, 160, 160, 162, 169, 161, 161, 158, 160, 168, 168, 159]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.925670408Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"":"downstairs","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"[2]R generation[0..12]: [87, 84, 84, 84, 87, 84, 84, 82, 84, 86, 86, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.925684398Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"":"downstairs","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:53:21.925698359Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"":"downstairs","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.925716479Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"":"downstairs","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.925729929Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"":"downstairs","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"Next flush: 170","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.925742949Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"":"downstairs","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.9257558Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"":"downstairs","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.92576927Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.92578166Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"client":"0","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.92579499Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"client":"1","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.925808231Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"client":"2","":"downstairs","session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-28T04:53:21.925821531Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.925835042Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"a07eadd3-5868-4a6a-a0cb-865b60519a0f is now active with session: fd31af8d-5057-4665-9d24-a8ea0297d2f1","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.925849902Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.925862402Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539,"session_id":"fd31af8d-5057-4665-9d24-a8ea0297d2f1"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:21.925874832Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 495 len 16384 data: 8 8 8 9 02/30 Write block 485 len 36864 data: 6 6 5 4 3 4 4 6 5 03/30 Read block 36 len 16384 04/30 Flush 05/30 Write block 304 len 8192 data: 6 6 06/30 Read block 76 len 24576 07/30 Read block 480 len 4096 08/30 Write block 331 len 16384 data: 8 7 7 10 09/30 Read block 117 len 36864 10/30 Flush 11/30 Write block 273 len 16384 data: 11 10 8 6 12/30 Write block 466 len 24576 data: 4 5 6 7 6 5 13/30 Read block 263 len 28672 14/30 Read block 588 len 40960 15/30 Flush 16/30 Write block 99 len 24576 data: 5 6 7 8 8 7 17/30 Write block 320 len 32768 data: 3 4 4 6 6 7 7 8 18/30 Write block 591 len 4096 data: 7 19/30 Read block 404 len 16384 20/30 Read block 372 len 28672 21/30 Write block 194 len 36864 data: 11 13 9 11 11 9 9 8 10 22/30 Read block 245 len 24576 23/30 Write block 545 len 8192 data: 8 8 24/30 Read block 143 len 32768 25/30 Write block 205 len 8192 data: 7 7 26/30 Read block 448 len 24576 27/30 Write block 217 len 8192 data: 5 6 28/30 Read block 6 len 40960 29/30 Write block 589 len 24576 data: 7 7 8 7 5 4 30/30 Read block 463 len 16384 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 6 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 6 Done Sent Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 1 3 Done 15 13 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:53:21.944559148Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1539} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 169 169 169 F F F 1 020-039 84 84 84 160 160 160 F F F 2 040-059 84 84 84 160 160 160 F F F 3 060-079 84 84 84 162 162 162 F F F 4 080-099 87 87 87 169 169 169 T T T 5 100-119 84 84 84 161 161 161 T T T 6 120-139 84 84 84 161 161 161 F F F 7 140-159 82 82 82 158 158 158 F F F 8 160-179 84 84 84 160 160 160 F F F 9 180-199 86 86 86 168 168 168 T T T 10 200-219 86 86 86 168 168 168 T T T 11 220-239 83 83 83 159 159 159 F F F 12 240-259 86 86 86 167 167 167 F F F 13 260-279 88 88 88 172 172 172 F F F 14 280-299 77 77 77 152 152 152 F F F 15 300-319 88 88 88 171 171 171 F F F 16 320-339 88 88 88 171 171 171 T T T 17 340-359 77 77 77 152 152 152 F F F 18 360-379 86 86 86 167 167 167 F F F 19 380-399 86 86 86 167 167 167 F F F 20 400-419 82 82 82 158 158 158 F F F 21 420-439 87 87 87 169 169 169 F F F 22 440-459 85 85 85 163 163 163 F F F 23 460-479 88 88 88 172 172 172 F F F 24 480-499 88 88 88 170 170 170 F F F 25 500-519 80 80 80 155 155 155 F F F 26 520-539 83 83 83 159 159 159 F F F 27 540-559 84 84 84 162 162 162 T T T 28 560-579 82 82 82 158 158 158 F F F 29 580-599 86 86 86 167 167 167 T T T Max gen: 88, Max flush: 172 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 169 169 169 F F F 1 020-039 84 84 84 160 160 160 F F F 2 040-059 84 84 84 160 160 160 F F F 3 060-079 84 84 84 162 162 162 F F F 4 080-099 89 89 89 173 173 173 F F F 5 100-119 89 89 89 173 173 173 F F F 6 120-139 84 84 84 161 161 161 F F F 7 140-159 82 82 82 158 158 158 F F F 8 160-179 84 84 84 160 160 160 F F F 9 180-199 89 89 89 173 173 173 F F F 10 200-219 89 89 89 173 173 173 F F F 11 220-239 83 83 83 159 159 159 F F F 12 240-259 86 86 86 167 167 167 F F F 13 260-279 88 88 88 172 172 172 F F F 14 280-299 77 77 77 152 152 152 F F F 15 300-319 88 88 88 171 171 171 F F F 16 320-339 89 89 89 173 173 173 F F F 17 340-359 77 77 77 152 152 152 F F F 18 360-379 86 86 86 167 167 167 F F F 19 380-399 86 86 86 167 167 167 F F F 20 400-419 82 82 82 158 158 158 F F F 21 420-439 87 87 87 169 169 169 F F F 22 440-459 85 85 85 163 163 163 F F F 23 460-479 88 88 88 172 172 172 F F F 24 480-499 88 88 88 170 170 170 F F F 25 500-519 80 80 80 155 155 155 F F F 26 520-539 83 83 83 159 159 159 F F F 27 540-559 89 89 89 173 173 173 F F F 28 560-579 82 82 82 158 158 158 F F F 29 580-599 89 89 89 173 173 173 F F F Max gen: 89, Max flush: 173 Begin loop 45 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.252718815Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.297584418Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.298226322Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.298265303Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.298280253Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.298295213Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547} {"msg":"Crucible 357c5e77-432a-41f2-82f6-3c455bf88600 has session id: 43c57ce1-a8ce-4808-a061-7705abd2ed72","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.298685272Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"Upstairs opts: Upstairs UUID: 357c5e77-432a-41f2-82f6-3c455bf88600, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.298728303Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"Crucible stats registered with UUID: 357c5e77-432a-41f2-82f6-3c455bf88600","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.298746143Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.330187945Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.330218976Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547} {"msg":"357c5e77-432a-41f2-82f6-3c455bf88600 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.330249557Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.330266247Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"client":"0","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.330294417Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"client":"0","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.330312198Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"client":"1","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.330329408Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"client":"1","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.330355659Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"client":"2","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.33039507Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"client":"2","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.33042876Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"":"io task","client":"2","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.330461961Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"":"io task","client":"0","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.330491442Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"":"io task","client":"1","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"ds_connection connected from Ok(127.0.0.1:32855)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.330511232Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"":"io task","client":"0","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"ds_connection connected from Ok(127.0.0.1:32866)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.330537683Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"":"io task","client":"1","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"ds_connection connected from Ok(127.0.0.1:35565)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.330670256Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"":"io task","client":"2","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.331178717Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"client":"1","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.331206177Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"client":"1","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.33134367Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"client":"0","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.331381421Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"client":"0","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.331407332Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"client":"2","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.331432382Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"client":"2","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.331818671Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"client":"1","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.331862222Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"client":"0","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.331945943Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"client":"2","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.332286971Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"client":"1","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.332328002Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"client":"1","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.332357812Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.332418414Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"client":"0","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.332448314Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"client":"0","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.332465115Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.332479325Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"client":"2","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.332494225Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"client":"2","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"[0]R flush_numbers[0..12]: [169, 160, 160, 162, 173, 173, 161, 158, 160, 173, 173, 159]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.332514576Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"":"downstairs","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"[0]R generation[0..12]: [87, 84, 84, 84, 89, 89, 84, 82, 84, 89, 89, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.332530126Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"":"downstairs","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.332545877Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"":"downstairs","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"[1]R flush_numbers[0..12]: [169, 160, 160, 162, 173, 173, 161, 158, 160, 173, 173, 159]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.332560777Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"":"downstairs","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"[1]R generation[0..12]: [87, 84, 84, 84, 89, 89, 84, 82, 84, 89, 89, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.332575657Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"":"downstairs","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.332590727Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"":"downstairs","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"[2]R flush_numbers[0..12]: [169, 160, 160, 162, 173, 173, 161, 158, 160, 173, 173, 159]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.332610298Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"":"downstairs","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"[2]R generation[0..12]: [87, 84, 84, 84, 89, 89, 84, 82, 84, 89, 89, 83]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.332625918Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"":"downstairs","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.332641428Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"":"downstairs","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.332655779Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"":"downstairs","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.332670379Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"":"downstairs","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"Next flush: 174","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.332685069Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"":"downstairs","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.33270638Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"":"downstairs","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.3327208Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.33273421Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"client":"0","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.332749511Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"client":"1","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.332763961Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"client":"2","":"downstairs","session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.332778591Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.332792242Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"357c5e77-432a-41f2-82f6-3c455bf88600 is now active with session: 43c57ce1-a8ce-4808-a061-7705abd2ed72","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-28T04:53:26.332806172Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.332823012Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547,"session_id":"43c57ce1-a8ce-4808-a061-7705abd2ed72"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:26.332839393Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 143 len 36864 data: 6 6 6 7 9 10 12 12 11 03/30 Read block 349 len 8192 04/30 Read block 288 len 8192 05/30 Flush 06/30 Write block 571 len 20480 data: 8 9 10 11 11 07/30 Flush 08/30 Write block 369 len 8192 data: 7 7 09/30 Read block 70 len 20480 10/30 Write block 221 len 32768 data: 8 8 9 7 6 4 6 4 11/30 Write block 174 len 16384 data: 7 7 7 6 12/30 Write block 471 len 28672 data: 6 4 4 8 8 10 9 13/30 Read block 549 len 8192 14/30 Write block 184 len 40960 data: 11 11 13 12 10 11 10 11 10 10 15/30 Write block 391 len 12288 data: 5 6 6 16/30 Read block 291 len 32768 17/30 Write block 445 len 4096 data: 5 18/30 Read block 323 len 28672 19/30 Write block 484 len 12288 data: 6 7 7 20/30 Read block 535 len 12288 21/30 Write block 498 len 40960 data: 10 9 13 10 8 10 8 6 8 9 22/30 Flush 23/30 Write block 109 len 40960 data: 3 4 5 6 6 7 8 7 7 6 24/30 Flush 25/30 Write block 166 len 8192 data: 6 6 26/30 Write block 130 len 12288 data: 5 5 6 27/30 Write block 163 len 8192 data: 6 6 28/30 Write block 245 len 16384 data: 8 11 11 12 29/30 Read block 388 len 8192 30/30 Read block 523 len 4096 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 2 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:53:26.351537228Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1547} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 169 169 169 F F F 1 020-039 84 84 84 160 160 160 F F F 2 040-059 84 84 84 160 160 160 F F F 3 060-079 84 84 84 162 162 162 F F F 4 080-099 89 89 89 173 173 173 F F F 5 100-119 90 90 90 178 178 178 F F F 6 120-139 84 84 84 161 161 161 T T T 7 140-159 90 90 90 175 175 175 F F F 8 160-179 90 90 90 177 177 177 T T T 9 180-199 90 90 90 177 177 177 F F F 10 200-219 89 89 89 173 173 173 F F F 11 220-239 90 90 90 177 177 177 F F F 12 240-259 86 86 86 167 167 167 T T T 13 260-279 88 88 88 172 172 172 F F F 14 280-299 77 77 77 152 152 152 F F F 15 300-319 88 88 88 171 171 171 F F F 16 320-339 89 89 89 173 173 173 F F F 17 340-359 77 77 77 152 152 152 F F F 18 360-379 90 90 90 177 177 177 F F F 19 380-399 90 90 90 177 177 177 F F F 20 400-419 82 82 82 158 158 158 F F F 21 420-439 87 87 87 169 169 169 F F F 22 440-459 90 90 90 177 177 177 F F F 23 460-479 90 90 90 177 177 177 F F F 24 480-499 90 90 90 177 177 177 F F F 25 500-519 90 90 90 177 177 177 F F F 26 520-539 83 83 83 159 159 159 F F F 27 540-559 89 89 89 173 173 173 F F F 28 560-579 90 90 90 176 176 176 F F F 29 580-599 89 89 89 173 173 173 F F F Max gen: 90, Max flush: 178 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 169 169 169 F F F 1 020-039 84 84 84 160 160 160 F F F 2 040-059 84 84 84 160 160 160 F F F 3 060-079 84 84 84 162 162 162 F F F 4 080-099 89 89 89 173 173 173 F F F 5 100-119 90 90 90 178 178 178 F F F 6 120-139 91 91 91 179 179 179 F F F 7 140-159 90 90 90 175 175 175 F F F 8 160-179 91 91 91 179 179 179 F F F 9 180-199 90 90 90 177 177 177 F F F 10 200-219 89 89 89 173 173 173 F F F 11 220-239 90 90 90 177 177 177 F F F 12 240-259 91 91 91 179 179 179 F F F 13 260-279 88 88 88 172 172 172 F F F 14 280-299 77 77 77 152 152 152 F F F 15 300-319 88 88 88 171 171 171 F F F 16 320-339 89 89 89 173 173 173 F F F 17 340-359 77 77 77 152 152 152 F F F 18 360-379 90 90 90 177 177 177 F F F 19 380-399 90 90 90 177 177 177 F F F 20 400-419 82 82 82 158 158 158 F F F 21 420-439 87 87 87 169 169 169 F F F 22 440-459 90 90 90 177 177 177 F F F 23 460-479 90 90 90 177 177 177 F F F 24 480-499 90 90 90 177 177 177 F F F 25 500-519 90 90 90 177 177 177 F F F 26 520-539 83 83 83 159 159 159 F F F 27 540-559 89 89 89 173 173 173 F F F 28 560-579 90 90 90 176 176 176 F F F 29 580-599 89 89 89 173 173 173 F F F Max gen: 91, Max flush: 179 Begin loop 46 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.6105605Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.654713117Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.655456254Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.655491014Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.655505375Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.655520005Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555} {"msg":"Crucible e17b5d0a-bc29-4d3a-ac99-336e9956f170 has session id: e466c599-08a8-4e52-afe1-b47f585b35cb","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.655953294Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"Upstairs opts: Upstairs UUID: e17b5d0a-bc29-4d3a-ac99-336e9956f170, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.656005735Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"Crucible stats registered with UUID: e17b5d0a-bc29-4d3a-ac99-336e9956f170","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.656025496Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.686696881Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.686723472Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555} {"msg":"e17b5d0a-bc29-4d3a-ac99-336e9956f170 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.686785713Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.686821024Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"client":"0","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.686851415Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"client":"0","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.686870225Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"client":"1","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.686887515Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"client":"1","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.686904096Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"client":"2","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.686920996Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"client":"2","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.686937626Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.686954847Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.686971837Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:63974)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.68709267Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:49523)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.687136201Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:53228)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.687167091Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.687865557Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"client":"0","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.687892987Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"client":"0","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.687908427Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"client":"1","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.687922238Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"client":"1","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.688057011Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"client":"2","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.688094651Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"client":"2","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.68849194Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"client":"1","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.688587362Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"client":"0","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.688758026Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"client":"2","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.689082113Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"client":"1","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.689113304Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"client":"1","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.689130594Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.689148894Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"client":"0","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.689164875Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"client":"0","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.689179515Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.689249987Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"client":"2","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.689271037Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"client":"2","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"[0]R flush_numbers[0..12]: [169, 160, 160, 162, 173, 178, 179, 175, 179, 177, 173, 177]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.689286767Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"":"downstairs","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"[0]R generation[0..12]: [87, 84, 84, 84, 89, 90, 91, 90, 91, 90, 89, 90]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.689302318Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"":"downstairs","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.689318078Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"":"downstairs","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"[1]R flush_numbers[0..12]: [169, 160, 160, 162, 173, 178, 179, 175, 179, 177, 173, 177]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.689332828Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"":"downstairs","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"[1]R generation[0..12]: [87, 84, 84, 84, 89, 90, 91, 90, 91, 90, 89, 90]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.689348159Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"":"downstairs","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.689363169Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"":"downstairs","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"[2]R flush_numbers[0..12]: [169, 160, 160, 162, 173, 178, 179, 175, 179, 177, 173, 177]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.689378349Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"":"downstairs","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"[2]R generation[0..12]: [87, 84, 84, 84, 89, 90, 91, 90, 91, 90, 89, 90]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.68939354Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"":"downstairs","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.68940887Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"":"downstairs","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.68942633Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"":"downstairs","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.689441521Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"":"downstairs","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"Next flush: 180","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:53:30.689456431Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"":"downstairs","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.689478521Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"":"downstairs","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.689492842Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.689506442Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"client":"0","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.689521473Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"client":"1","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.689550063Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"client":"2","":"downstairs","session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.689565603Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.689582654Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"e17b5d0a-bc29-4d3a-ac99-336e9956f170 is now active with session: e466c599-08a8-4e52-afe1-b47f585b35cb","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.689597054Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.689610974Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555,"session_id":"e466c599-08a8-4e52-afe1-b47f585b35cb"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:30.689624525Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 555 len 4096 data: 10 03/30 Read block 211 len 36864 04/30 Write block 504 len 40960 data: 9 7 9 10 8 8 7 6 8 8 05/30 Write block 214 len 4096 data: 8 06/30 Read block 35 len 36864 07/30 Read block 517 len 36864 08/30 Read block 73 len 28672 09/30 Write block 298 len 12288 data: 6 7 7 10/30 Write block 49 len 20480 data: 8 10 9 9 9 11/30 Read block 574 len 40960 12/30 Read block 502 len 24576 13/30 Read block 220 len 4096 14/30 Write block 214 len 40960 data: 9 9 9 6 7 7 6 9 9 10 15/30 Write block 110 len 40960 data: 5 6 7 7 8 9 8 8 7 6 16/30 Write block 197 len 4096 data: 12 17/30 Read block 576 len 36864 18/30 Flush 19/30 Write block 205 len 8192 data: 8 8 20/30 Write block 442 len 16384 data: 5 5 5 6 21/30 Write block 43 len 12288 data: 8 8 8 22/30 Flush 23/30 Write block 547 len 28672 data: 8 8 8 9 8 10 10 24/30 Write block 51 len 24576 data: 10 10 10 12 11 12 25/30 Flush 26/30 Write block 503 len 36864 data: 11 10 8 10 11 9 9 8 7 27/30 Read block 203 len 8192 28/30 Read block 197 len 4096 29/30 Write block 494 len 36864 data: 7 9 9 9 11 10 14 11 9 30/30 Read block 60 len 32768 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 9 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 9 Done Done Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 5 4 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:53:30.708741609Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1555} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 169 169 169 F F F 1 020-039 84 84 84 160 160 160 F F F 2 040-059 92 92 92 183 183 183 F F F 3 060-079 84 84 84 162 162 162 F F F 4 080-099 89 89 89 173 173 173 F F F 5 100-119 92 92 92 181 181 181 F F F 6 120-139 91 91 91 179 179 179 F F F 7 140-159 90 90 90 175 175 175 F F F 8 160-179 91 91 91 179 179 179 F F F 9 180-199 92 92 92 181 181 181 F F F 10 200-219 92 92 92 182 182 182 F F F 11 220-239 92 92 92 181 181 181 F F F 12 240-259 91 91 91 179 179 179 F F F 13 260-279 88 88 88 172 172 172 F F F 14 280-299 92 92 92 181 181 181 F F F 15 300-319 92 92 92 181 181 181 F F F 16 320-339 89 89 89 173 173 173 F F F 17 340-359 77 77 77 152 152 152 F F F 18 360-379 90 90 90 177 177 177 F F F 19 380-399 90 90 90 177 177 177 F F F 20 400-419 82 82 82 158 158 158 F F F 21 420-439 87 87 87 169 169 169 F F F 22 440-459 92 92 92 182 182 182 F F F 23 460-479 90 90 90 177 177 177 F F F 24 480-499 90 90 90 177 177 177 T T T 25 500-519 92 92 92 181 181 181 T T T 26 520-539 83 83 83 159 159 159 F F F 27 540-559 92 92 92 183 183 183 F F F 28 560-579 90 90 90 176 176 176 F F F 29 580-599 89 89 89 173 173 173 F F F Max gen: 92, Max flush: 183 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 169 169 169 F F F 1 020-039 84 84 84 160 160 160 F F F 2 040-059 92 92 92 183 183 183 F F F 3 060-079 84 84 84 162 162 162 F F F 4 080-099 89 89 89 173 173 173 F F F 5 100-119 92 92 92 181 181 181 F F F 6 120-139 91 91 91 179 179 179 F F F 7 140-159 90 90 90 175 175 175 F F F 8 160-179 91 91 91 179 179 179 F F F 9 180-199 92 92 92 181 181 181 F F F 10 200-219 92 92 92 182 182 182 F F F 11 220-239 92 92 92 181 181 181 F F F 12 240-259 91 91 91 179 179 179 F F F 13 260-279 88 88 88 172 172 172 F F F 14 280-299 92 92 92 181 181 181 F F F 15 300-319 92 92 92 181 181 181 F F F 16 320-339 89 89 89 173 173 173 F F F 17 340-359 77 77 77 152 152 152 F F F 18 360-379 90 90 90 177 177 177 F F F 19 380-399 90 90 90 177 177 177 F F F 20 400-419 82 82 82 158 158 158 F F F 21 420-439 87 87 87 169 169 169 F F F 22 440-459 92 92 92 182 182 182 F F F 23 460-479 90 90 90 177 177 177 F F F 24 480-499 93 93 93 184 184 184 F F F 25 500-519 93 93 93 184 184 184 F F F 26 520-539 83 83 83 159 159 159 F F F 27 540-559 92 92 92 183 183 183 F F F 28 560-579 90 90 90 176 176 176 F F F 29 580-599 89 89 89 173 173 173 F F F Max gen: 93, Max flush: 184 Begin loop 47 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:34.954731867Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:34.998933206Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:34.99954497Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:34.99958114Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:34.999596431Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:34.999612121Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563} {"msg":"Crucible 37625a55-7a13-4a3a-91a2-baed3667d400 has session id: 9af36d05-d835-41a9-acdc-8044a13feb31","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.000073361Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"Upstairs opts: Upstairs UUID: 37625a55-7a13-4a3a-91a2-baed3667d400, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.000120802Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"Crucible stats registered with UUID: 37625a55-7a13-4a3a-91a2-baed3667d400","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.000148983Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.031489843Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.031524893Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563} {"msg":"37625a55-7a13-4a3a-91a2-baed3667d400 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.031576174Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.031596135Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"client":"0","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.031625546Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"client":"0","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.031643316Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"client":"1","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.031660146Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"client":"1","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.031676777Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"client":"2","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.031693527Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"client":"2","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.031709888Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"":"io task","client":"2","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.031727048Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"":"io task","client":"0","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.031757898Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"":"io task","client":"1","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"ds_connection connected from Ok(127.0.0.1:62511)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.031873211Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"":"io task","client":"2","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"ds_connection connected from Ok(127.0.0.1:64999)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.031915962Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"":"io task","client":"0","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"ds_connection connected from Ok(127.0.0.1:42411)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.032053105Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"":"io task","client":"1","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.032681878Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"client":"1","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.032719209Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"client":"1","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.032780881Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"client":"0","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.032802371Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"client":"0","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.032817141Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"client":"2","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.032830552Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"client":"2","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.033366713Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"client":"0","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.033395924Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"client":"1","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.033411924Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"client":"2","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.033959336Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"client":"2","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.033985497Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"client":"2","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.034000087Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.034087699Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"client":"1","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.03411028Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"client":"1","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.03412457Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.034137Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"client":"0","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.034153601Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"client":"0","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"[0]R flush_numbers[0..12]: [169, 160, 183, 162, 173, 181, 179, 175, 179, 181, 182, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.034168061Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"":"downstairs","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"[0]R generation[0..12]: [87, 84, 92, 84, 89, 92, 91, 90, 91, 92, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.034182071Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"":"downstairs","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.034196361Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"":"downstairs","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"[1]R flush_numbers[0..12]: [169, 160, 183, 162, 173, 181, 179, 175, 179, 181, 182, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.034210212Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"":"downstairs","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"[1]R generation[0..12]: [87, 84, 92, 84, 89, 92, 91, 90, 91, 92, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.034223912Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"":"downstairs","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.034237862Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"":"downstairs","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"[2]R flush_numbers[0..12]: [169, 160, 183, 162, 173, 181, 179, 175, 179, 181, 182, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.034251443Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"":"downstairs","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"[2]R generation[0..12]: [87, 84, 92, 84, 89, 92, 91, 90, 91, 92, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.034265073Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"":"downstairs","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.034279763Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"":"downstairs","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.034303614Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.034338814Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"":"downstairs","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"Next flush: 185","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.034360215Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"":"downstairs","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.034374165Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"":"downstairs","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.034390746Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.034403346Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"client":"0","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.034417066Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"client":"1","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.034430316Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"client":"2","":"downstairs","session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.034443597Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.034456027Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"37625a55-7a13-4a3a-91a2-baed3667d400 is now active with session: 9af36d05-d835-41a9-acdc-8044a13feb31","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.034469077Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.034481197Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563,"session_id":"9af36d05-d835-41a9-acdc-8044a13feb31"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:35.034493498Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 311 len 4096 data: 3 02/30 Write block 38 len 36864 data: 9 11 11 9 9 9 9 9 9 03/30 Write block 27 len 24576 data: 8 9 8 4 5 5 04/30 Read block 434 len 12288 05/30 Read block 249 len 24576 06/30 Write block 155 len 4096 data: 10 07/30 Read block 214 len 32768 08/30 Write block 245 len 4096 data: 9 09/30 Write block 93 len 12288 data: 8 7 9 10/30 Read block 598 len 8192 11/30 Read block 455 len 8192 12/30 Write block 240 len 40960 data: 9 8 8 9 8 10 12 12 13 12 13/30 Write block 570 len 24576 data: 9 9 10 11 12 12 14/30 Read block 174 len 16384 15/30 Read block 289 len 16384 16/30 Flush 17/30 Write block 340 len 40960 data: 9 8 8 8 9 9 8 10 9 9 18/30 Read block 91 len 24576 19/30 Read block 104 len 16384 20/30 Write block 520 len 12288 data: 6 7 5 21/30 Write block 128 len 20480 data: 7 5 6 6 7 22/30 Write block 274 len 40960 data: 11 9 7 7 7 4 3 3 5 5 23/30 Write block 410 len 16384 data: 10 8 7 8 24/30 Read block 127 len 12288 25/30 Read block 476 len 40960 26/30 Read block 204 len 36864 27/30 Read block 478 len 24576 28/30 Write block 263 len 24576 data: 4 6 6 8 9 9 29/30 Read block 552 len 8192 30/30 Read block 184 len 4096 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 10 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:53:35.053311606Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1563} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 169 169 169 F F F 1 020-039 94 94 94 185 185 185 F F F 2 040-059 94 94 94 185 185 185 F F F 3 060-079 84 84 84 162 162 162 F F F 4 080-099 94 94 94 185 185 185 F F F 5 100-119 92 92 92 181 181 181 F F F 6 120-139 91 91 91 179 179 179 T T T 7 140-159 94 94 94 185 185 185 F F F 8 160-179 91 91 91 179 179 179 F F F 9 180-199 92 92 92 181 181 181 F F F 10 200-219 92 92 92 182 182 182 F F F 11 220-239 92 92 92 181 181 181 F F F 12 240-259 94 94 94 185 185 185 F F F 13 260-279 88 88 88 172 172 172 T T T 14 280-299 92 92 92 181 181 181 T T T 15 300-319 94 94 94 185 185 185 F F F 16 320-339 89 89 89 173 173 173 F F F 17 340-359 77 77 77 152 152 152 T T T 18 360-379 90 90 90 177 177 177 F F F 19 380-399 90 90 90 177 177 177 F F F 20 400-419 82 82 82 158 158 158 T T T 21 420-439 87 87 87 169 169 169 F F F 22 440-459 92 92 92 182 182 182 F F F 23 460-479 90 90 90 177 177 177 F F F 24 480-499 93 93 93 184 184 184 F F F 25 500-519 93 93 93 184 184 184 F F F 26 520-539 83 83 83 159 159 159 T T T 27 540-559 92 92 92 183 183 183 F F F 28 560-579 94 94 94 185 185 185 F F F 29 580-599 89 89 89 173 173 173 F F F Max gen: 94, Max flush: 185 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 169 169 169 F F F 1 020-039 94 94 94 185 185 185 F F F 2 040-059 94 94 94 185 185 185 F F F 3 060-079 84 84 84 162 162 162 F F F 4 080-099 94 94 94 185 185 185 F F F 5 100-119 92 92 92 181 181 181 F F F 6 120-139 95 95 95 186 186 186 F F F 7 140-159 94 94 94 185 185 185 F F F 8 160-179 91 91 91 179 179 179 F F F 9 180-199 92 92 92 181 181 181 F F F 10 200-219 92 92 92 182 182 182 F F F 11 220-239 92 92 92 181 181 181 F F F 12 240-259 94 94 94 185 185 185 F F F 13 260-279 95 95 95 186 186 186 F F F 14 280-299 95 95 95 186 186 186 F F F 15 300-319 94 94 94 185 185 185 F F F 16 320-339 89 89 89 173 173 173 F F F 17 340-359 95 95 95 186 186 186 F F F 18 360-379 90 90 90 177 177 177 F F F 19 380-399 90 90 90 177 177 177 F F F 20 400-419 95 95 95 186 186 186 F F F 21 420-439 87 87 87 169 169 169 F F F 22 440-459 92 92 92 182 182 182 F F F 23 460-479 90 90 90 177 177 177 F F F 24 480-499 93 93 93 184 184 184 F F F 25 500-519 93 93 93 184 184 184 F F F 26 520-539 95 95 95 186 186 186 F F F 27 540-559 92 92 92 183 183 183 F F F 28 560-579 94 94 94 185 185 185 F F F 29 580-599 89 89 89 173 173 173 F F F Max gen: 95, Max flush: 186 Begin loop 48 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.322561759Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.36640112Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.367000053Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.367036954Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.367051374Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.367065734Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571} {"msg":"Crucible 37319e23-3488-4355-bc01-2760dd88952a has session id: 1cdffa9f-4830-488b-aee1-7f14eb241a86","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.367452023Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"Upstairs opts: Upstairs UUID: 37319e23-3488-4355-bc01-2760dd88952a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.367498234Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"Crucible stats registered with UUID: 37319e23-3488-4355-bc01-2760dd88952a","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.367524294Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.399357645Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.399389885Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571} {"msg":"37319e23-3488-4355-bc01-2760dd88952a active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.399430016Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.399447517Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"client":"0","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.399476067Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"client":"0","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.399494628Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"client":"1","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.399511848Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"client":"1","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.399528488Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"client":"2","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.399545519Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"client":"2","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.399562329Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.39957962Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.39959683Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"ds_connection connected from Ok(127.0.0.1:53012)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.399795204Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"ds_connection connected from Ok(127.0.0.1:54674)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.399835495Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"ds_connection connected from Ok(127.0.0.1:51010)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.399865156Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.400578321Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"client":"0","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.400607802Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"client":"0","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.400630362Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"client":"2","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.400658343Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"client":"2","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.400687174Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"client":"1","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.400714794Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"client":"1","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.401284697Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"client":"1","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.401315477Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"client":"2","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.401332578Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"client":"0","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.401848759Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"client":"0","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.401877019Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"client":"0","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.40189323Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.401950731Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"client":"1","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.401987422Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"client":"1","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.402004802Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.402018753Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"client":"2","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.402033513Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"client":"2","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"[0]R flush_numbers[0..12]: [169, 185, 185, 162, 185, 181, 186, 185, 179, 181, 182, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.402049153Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"":"downstairs","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"[0]R generation[0..12]: [87, 94, 94, 84, 94, 92, 95, 94, 91, 92, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.402068323Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"":"downstairs","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.402084254Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"":"downstairs","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"[1]R flush_numbers[0..12]: [169, 185, 185, 162, 185, 181, 186, 185, 179, 181, 182, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.402099394Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"":"downstairs","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"[1]R generation[0..12]: [87, 94, 94, 84, 94, 92, 95, 94, 91, 92, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.402114464Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"":"downstairs","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.402129665Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"":"downstairs","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"[2]R flush_numbers[0..12]: [169, 185, 185, 162, 185, 181, 186, 185, 179, 181, 182, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.402144825Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"":"downstairs","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"[2]R generation[0..12]: [87, 94, 94, 84, 94, 92, 95, 94, 91, 92, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.402160205Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"":"downstairs","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.402175806Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"":"downstairs","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.402190416Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"":"downstairs","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:53:39.402205446Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"":"downstairs","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"Next flush: 187","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.402226077Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"":"downstairs","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.402240657Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"":"downstairs","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.402255258Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.402268748Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"client":"0","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.402285308Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"client":"1","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.402300099Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"client":"2","":"downstairs","session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.402315189Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.402331219Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"37319e23-3488-4355-bc01-2760dd88952a is now active with session: 1cdffa9f-4830-488b-aee1-7f14eb241a86","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.40234512Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.40235867Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571,"session_id":"1cdffa9f-4830-488b-aee1-7f14eb241a86"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:39.40237218Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 376 len 28672 data: 7 8 7 7 6 6 6 02/30 Write block 55 len 28672 data: 12 13 12 12 10 11 10 03/30 Write block 314 len 24576 data: 7 7 4 3 3 4 04/30 Read block 344 len 8192 05/30 Write block 551 len 20480 data: 9 11 11 10 11 06/30 Flush 07/30 Write block 558 len 8192 data: 8 9 08/30 Read block 463 len 36864 09/30 Write block 573 len 36864 data: 12 13 13 9 8 8 7 5 5 10/30 Read block 179 len 40960 11/30 Write block 487 len 20480 data: 6 5 4 5 5 12/30 Write block 431 len 40960 data: 8 7 8 8 8 8 5 6 6 7 13/30 Write block 520 len 28672 data: 7 8 6 8 11 10 12 14/30 Read block 518 len 24576 15/30 Read block 540 len 24576 16/30 Read block 140 len 8192 17/30 Write block 153 len 40960 data: 10 10 11 8 7 7 5 6 6 6 18/30 Read block 48 len 32768 19/30 Read block 289 len 28672 20/30 Write block 276 len 40960 data: 8 8 8 5 4 4 6 6 7 9 21/30 Write block 262 len 20480 data: 6 5 7 7 9 22/30 Write block 201 len 8192 data: 9 11 23/30 Read block 567 len 12288 24/30 Read block 341 len 8192 25/30 Write block 544 len 20480 data: 5 9 9 9 9 26/30 Read block 99 len 20480 27/30 Read block 125 len 24576 28/30 Write block 92 len 28672 data: 9 9 8 10 7 4 5 29/30 Read block 131 len 28672 30/30 Write block 482 len 24576 data: 8 6 7 8 8 7 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 2 Done Done Done false 1007 Acked Read 9 Done Done Done false 1008 Acked Write 9 Done Done Done false 1009 Acked Read 10 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Write 7 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 23 23 23 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1005 1005 1005 Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:53:39.422227421Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1571} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 169 169 169 F F F 1 020-039 94 94 94 185 185 185 F F F 2 040-059 96 96 96 187 187 187 F F F 3 060-079 96 96 96 187 187 187 F F F 4 080-099 94 94 94 185 185 185 T T T 5 100-119 92 92 92 181 181 181 F F F 6 120-139 95 95 95 186 186 186 F F F 7 140-159 94 94 94 185 185 185 T T T 8 160-179 91 91 91 179 179 179 T T T 9 180-199 92 92 92 181 181 181 F F F 10 200-219 92 92 92 182 182 182 T T T 11 220-239 92 92 92 181 181 181 F F F 12 240-259 94 94 94 185 185 185 F F F 13 260-279 95 95 95 186 186 186 T T T 14 280-299 95 95 95 186 186 186 T T T 15 300-319 96 96 96 187 187 187 F F F 16 320-339 89 89 89 173 173 173 F F F 17 340-359 95 95 95 186 186 186 F F F 18 360-379 96 96 96 187 187 187 F F F 19 380-399 96 96 96 187 187 187 F F F 20 400-419 95 95 95 186 186 186 F F F 21 420-439 87 87 87 169 169 169 T T T 22 440-459 92 92 92 182 182 182 T T T 23 460-479 90 90 90 177 177 177 F F F 24 480-499 93 93 93 184 184 184 T T T 25 500-519 93 93 93 184 184 184 F F F 26 520-539 95 95 95 186 186 186 T T T 27 540-559 96 96 96 187 187 187 T T T 28 560-579 94 94 94 185 185 185 T T T 29 580-599 89 89 89 173 173 173 T T T Max gen: 96, Max flush: 187 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 169 169 169 F F F 1 020-039 94 94 94 185 185 185 F F F 2 040-059 96 96 96 187 187 187 F F F 3 060-079 96 96 96 187 187 187 F F F 4 080-099 97 97 97 188 188 188 F F F 5 100-119 92 92 92 181 181 181 F F F 6 120-139 95 95 95 186 186 186 F F F 7 140-159 97 97 97 188 188 188 F F F 8 160-179 97 97 97 188 188 188 F F F 9 180-199 92 92 92 181 181 181 F F F 10 200-219 97 97 97 188 188 188 F F F 11 220-239 92 92 92 181 181 181 F F F 12 240-259 94 94 94 185 185 185 F F F 13 260-279 97 97 97 188 188 188 F F F 14 280-299 97 97 97 188 188 188 F F F 15 300-319 96 96 96 187 187 187 F F F 16 320-339 89 89 89 173 173 173 F F F 17 340-359 95 95 95 186 186 186 F F F 18 360-379 96 96 96 187 187 187 F F F 19 380-399 96 96 96 187 187 187 F F F 20 400-419 95 95 95 186 186 186 F F F 21 420-439 97 97 97 188 188 188 F F F 22 440-459 97 97 97 188 188 188 F F F 23 460-479 90 90 90 177 177 177 F F F 24 480-499 97 97 97 188 188 188 F F F 25 500-519 93 93 93 184 184 184 F F F 26 520-539 97 97 97 188 188 188 F F F 27 540-559 97 97 97 188 188 188 F F F 28 560-579 97 97 97 188 188 188 F F F 29 580-599 97 97 97 188 188 188 F F F Max gen: 97, Max flush: 188 Begin loop 49 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.732922093Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:53:43.775532837Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.775894015Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.807697225Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.808278647Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.808310188Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.808323838Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.808338599Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579} {"msg":"Crucible de16076d-c0b4-4ea1-804c-4f8767e89e1c has session id: dfa5ad1d-6fb9-4c0d-812e-a3baa9357461","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.808739927Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"Upstairs opts: Upstairs UUID: de16076d-c0b4-4ea1-804c-4f8767e89e1c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.808785568Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"Crucible stats registered with UUID: de16076d-c0b4-4ea1-804c-4f8767e89e1c","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.808815219Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.839575816Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.839601927Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579} {"msg":"de16076d-c0b4-4ea1-804c-4f8767e89e1c active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.839657848Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.839676478Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"client":"0","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.839700649Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"client":"0","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.839716009Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"client":"1","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.83973028Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"client":"1","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.83974517Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"client":"2","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.83975959Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"client":"2","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.839784291Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"":"io task","client":"2","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.839813691Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"":"io task","client":"0","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.839838532Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"":"io task","client":"1","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"ds_connection connected from Ok(127.0.0.1:63346)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.839867163Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"":"io task","client":"2","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"ds_connection connected from Ok(127.0.0.1:64741)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.839905313Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"":"io task","client":"0","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"ds_connection connected from Ok(127.0.0.1:64672)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.840080937Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"":"io task","client":"1","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.840692761Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"client":"2","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.840733241Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"client":"2","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.840751442Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"client":"1","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.840765992Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"client":"1","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.840787612Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"client":"0","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.840802573Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"client":"0","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.841329564Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"client":"2","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.841408556Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"client":"1","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.841505538Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"client":"0","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.841936237Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"client":"1","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.841973588Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"client":"1","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.842001909Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.842026879Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"client":"2","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.84205651Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"client":"2","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.84208112Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.842106871Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"client":"0","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.842131612Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"client":"0","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"[0]R flush_numbers[0..12]: [169, 185, 187, 187, 188, 181, 186, 188, 188, 181, 188, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.842156532Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"":"downstairs","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"[0]R generation[0..12]: [87, 94, 96, 96, 97, 92, 95, 97, 97, 92, 97, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.842181823Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"":"downstairs","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.842206323Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"":"downstairs","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"[1]R flush_numbers[0..12]: [169, 185, 187, 187, 188, 181, 186, 188, 188, 181, 188, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.842229654Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"":"downstairs","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"[1]R generation[0..12]: [87, 94, 96, 96, 97, 92, 95, 97, 97, 92, 97, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.842254684Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"":"downstairs","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.842282665Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"":"downstairs","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"[2]R flush_numbers[0..12]: [169, 185, 187, 187, 188, 181, 186, 188, 188, 181, 188, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.842308636Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"":"downstairs","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"[2]R generation[0..12]: [87, 94, 96, 96, 97, 92, 95, 97, 97, 92, 97, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.842336596Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"":"downstairs","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v"Wait for a query_work_queue command to finish before sending IO :0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.842376097Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"":"downstairs","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.842407148Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"":"downstairs","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.842436798Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"":"downstairs","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"Next flush: 189","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.842466299Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"":"downstairs","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.84249889Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"":"downstairs","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.84252874Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.842560841Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"client":"0","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.842580931Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"client":"1","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.842595262Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"client":"2","":"downstairs","session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.842609172Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.842621632Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"de16076d-c0b4-4ea1-804c-4f8767e89e1c is now active with session: dfa5ad1d-6fb9-4c0d-812e-a3baa9357461","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.842634832Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.842647103Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579,"session_id":"dfa5ad1d-6fb9-4c0d-812e-a3baa9357461"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:43.842660233Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 539 len 28672 data: 7 5 3 5 5 6 10 02/30 Read block 485 len 16384 03/30 Flush 04/30 Write block 254 len 4096 data: 10 05/30 Read block 23 len 40960 06/30 Read block 313 len 40960 07/30 Write block 499 len 4096 data: 11 08/30 Read block 453 len 4096 09/30 Write block 107 len 36864 data: 4 3 4 6 7 8 8 9 10 10/30 Write block 421 len 24576 data: 5 6 6 6 6 6 11/30 Write block 78 len 24576 data: 9 8 5 7 5 3 12/30 Read block 62 len 16384 13/30 Read block 270 len 20480 14/30 Read block 50 len 16384 15/30 Write block 151 len 36864 data: 12 12 11 11 12 9 8 8 6 16/30 Write block 420 len 12288 data: 4 6 7 17/30 Write block 440 len 40960 data: 8 9 6 6 6 7 5 5 6 6 18/30 Write block 147 len 4096 data: 10 19/30 Flush 20/30 Write block 91 len 40960 data: 10 10 10 9 11 8 5 6 6 7 21/30 Read block 580 len 4096 22/30 Read block 286 len 24576 23/30 Read block 25 len 28672 24/30 Write block 399 len 16384 data: 9 8 9 8 25/30 Write block 571 len 4096 data: 10 26/30 Read block 371 len 40960 27/30 Write block 286 len 24576 data: 9 8 6 6 6 4 28/30 Write block 442 len 8192 data: 7 7 29/30 Write block 35 len 4096 data: 6 30/30 Write block 483 len 40960 data: 7 8 9 9 8 6 5 6 6 7 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 10 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:53:43.860410118Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1579} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 169 169 169 F F F 1 020-039 94 94 94 185 185 185 T T T 2 040-059 96 96 96 187 187 187 F F F 3 060-079 98 98 98 190 190 190 F F F 4 080-099 98 98 98 190 190 190 T T T 5 100-119 98 98 98 190 190 190 T T T 6 120-139 95 95 95 186 186 186 F F F 7 140-159 98 98 98 190 190 190 F F F 8 160-179 97 97 97 188 188 188 F F F 9 180-199 92 92 92 181 181 181 F F F 10 200-219 97 97 97 188 188 188 F F F 11 220-239 92 92 92 181 181 181 F F F 12 240-259 98 98 98 190 190 190 F F F 13 260-279 97 97 97 188 188 188 F F F 14 280-299 97 97 97 188 188 188 T T T 15 300-319 96 96 96 187 187 187 F F F 16 320-339 89 89 89 173 173 173 F F F 17 340-359 95 95 95 186 186 186 F F F 18 360-379 96 96 96 187 187 187 F F F 19 380-399 96 96 96 187 187 187 T T T 20 400-419 95 95 95 186 186 186 T T T 21 420-439 98 98 98 190 190 190 F F F 22 440-459 98 98 98 190 190 190 T T T 23 460-479 90 90 90 177 177 177 F F F 24 480-499 98 98 98 190 190 190 T T T 25 500-519 93 93 93 184 184 184 F F F 26 520-539 98 98 98 189 189 189 F F F 27 540-559 98 98 98 189 189 189 F F F 28 560-579 97 97 97 188 188 188 T T T 29 580-599 97 97 97 188 188 188 F F F Max gen: 98, Max flush: 190 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 169 169 169 F F F 1 020-039 99 99 99 191 191 191 F F F 2 040-059 96 96 96 187 187 187 F F F 3 060-079 98 98 98 190 190 190 F F F 4 080-099 99 99 99 191 191 191 F F F 5 100-119 99 99 99 191 191 191 F F F 6 120-139 95 95 95 186 186 186 F F F 7 140-159 98 98 98 190 190 190 F F F 8 160-179 97 97 97 188 188 188 F F F 9 180-199 92 92 92 181 181 181 F F F 10 200-219 97 97 97 188 188 188 F F F 11 220-239 92 92 92 181 181 181 F F F 12 240-259 98 98 98 190 190 190 F F F 13 260-279 97 97 97 188 188 188 F F F 14 280-299 99 99 99 191 191 191 F F F 15 300-319 96 96 96 187 187 187 F F F 16 320-339 89 89 89 173 173 173 F F F 17 340-359 95 95 95 186 186 186 F F F 18 360-379 96 96 96 187 187 187 F F F 19 380-399 99 99 99 191 191 191 F F F 20 400-419 99 99 99 191 191 191 F F F 21 420-439 98 98 98 190 190 190 F F F 22 440-459 99 99 99 191 191 191 F F F 23 460-479 90 90 90 177 177 177 F F F 24 480-499 99 99 99 191 191 191 F F F 25 500-519 93 93 93 184 184 184 F F F 26 520-539 98 98 98 189 189 189 F F F 27 540-559 98 98 98 189 189 189 F F F 28 560-579 99 99 99 191 191 191 F F F 29 580-599 97 97 97 188 188 188 F F F Max gen: 99, Max flush: 191 Begin loop 50 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.182127889Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.226985283Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.227656247Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.227693228Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.227708988Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.227723619Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587} {"msg":"Crucible c68a2e80-feac-4b13-97ab-25f74835ef2a has session id: 457aa521-4437-41ee-998d-8de009c65548","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.228144578Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"Upstairs opts: Upstairs UUID: c68a2e80-feac-4b13-97ab-25f74835ef2a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.228181459Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"Crucible stats registered with UUID: c68a2e80-feac-4b13-97ab-25f74835ef2a","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.228198469Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.259680292Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.259731343Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587} {"msg":"c68a2e80-feac-4b13-97ab-25f74835ef2a active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.259766674Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.259796754Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"client":"0","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.259845466Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"client":"0","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.259885196Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"client":"1","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.259921007Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"client":"1","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.259955108Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"client":"2","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.259989029Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"client":"2","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.260007459Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"":"io task","client":"2","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.260025199Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"":"io task","client":"0","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.26004204Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"":"io task","client":"1","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"ds_connection connected from Ok(127.0.0.1:35923)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.26005946Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"":"io task","client":"0","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"ds_connection connected from Ok(127.0.0.1:34128)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.260082181Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"":"io task","client":"2","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"ds_connection connected from Ok(127.0.0.1:60099)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.260277295Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"":"io task","client":"1","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.260840977Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"client":"1","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.260882718Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"client":"1","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.260922149Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"client":"0","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.26095645Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"client":"0","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.26097507Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"client":"2","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.26098869Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"client":"2","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.261514462Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"client":"1","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.261565103Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"client":"2","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.261622464Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"client":"0","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.262072154Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"client":"1","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.262106415Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"client":"1","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.262122545Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.262192816Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"client":"2","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.262212687Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"client":"2","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.262227007Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.262239618Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"client":"0","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.262253268Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"client":"0","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"[0]R flush_numbers[0..12]: [169, 191, 187, 190, 191, 191, 186, 190, 188, 181, 188, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.262267188Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"":"downstairs","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"[0]R generation[0..12]: [87, 99, 96, 98, 99, 99, 95, 98, 97, 92, 97, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.262285038Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"":"downstairs","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.262299899Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"":"downstairs","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"[1]R flush_numbers[0..12]: [169, 191, 187, 190, 191, 191, 186, 190, 188, 181, 188, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.262313869Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"":"downstairs","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"[1]R generation[0..12]: [87, 99, 96, 98, 99, 99, 95, 98, 97, 92, 97, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.262327739Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"":"downstairs","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.26234174Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"":"downstairs","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"[2]R flush_numbers[0..12]: [169, 191, 187, 190, 191, 191, 186, 190, 188, 181, 188, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.26235557Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"":"downstairs","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"[2]R generation[0..12]: [87, 99, 96, 98, 99, 99, 95, 98, 97, 92, 97, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.26236906Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"":"downstairs","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.26238297Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"":"downstairs","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.262396001Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"":"downstairs","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.262409311Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"":"downstairs","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"Next flush: 192","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.262422371Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"":"downstairs","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"Wait for a query_work_queue command to finish before sending IO All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.262443552Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"":"downstairs","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.262456982Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.262469052Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"client":"0","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.262483943Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"client":"1","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.262497163Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"client":"2","":"downstairs","session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.262510423Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.262522884Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"c68a2e80-feac-4b13-97ab-25f74835ef2a is now active with session: 457aa521-4437-41ee-998d-8de009c65548","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.262535554Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.262547714Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587,"session_id":"457aa521-4437-41ee-998d-8de009c65548"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:48.262560154Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 439 len 20480 data: 7 9 10 8 8 02/30 Write block 165 len 4096 data: 5 03/30 Read block 78 len 16384 04/30 Flush 05/30 Write block 490 len 8192 data: 7 7 06/30 Write block 212 len 28672 data: 10 9 10 10 10 7 8 07/30 Write block 141 len 24576 data: 7 7 7 7 7 8 08/30 Read block 23 len 8192 09/30 Write block 361 len 12288 data: 9 8 8 10/30 Write block 57 len 28672 data: 13 13 11 12 11 7 6 11/30 Read block 587 len 24576 12/30 Write block 172 len 20480 data: 8 7 8 8 8 13/30 Read block 103 len 28672 14/30 Write block 37 len 24576 data: 7 10 12 12 10 10 15/30 Read block 248 len 20480 16/30 Write block 509 len 40960 data: 10 9 8 9 9 8 9 8 9 9 17/30 Flush 18/30 Write block 392 len 8192 data: 7 7 19/30 Read block 306 len 32768 20/30 Read block 446 len 8192 21/30 Read block 100 len 28672 22/30 Write block 545 len 32768 data: 11 10 10 10 9 10 10 12 23/30 Read block 550 len 16384 24/30 Write block 17 len 12288 data: 7 8 11 25/30 Write block 108 len 28672 data: 4 5 7 8 9 9 10 26/30 Read block 131 len 24576 27/30 Read block 471 len 28672 28/30 Write block 377 len 28672 data: 9 8 8 7 7 7 5 29/30 Write block 200 len 36864 data: 10 10 12 9 7 9 9 6 7 30/30 Read block 402 len 4096 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 2 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 9 Done Done Sent false 1029 Acked Read 1 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 12 13 11 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:53:48.281902724Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1587} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 169 169 169 T T T 1 020-039 100 100 100 193 193 193 F F F 2 040-059 100 100 100 193 193 193 F F F 3 060-079 100 100 100 193 193 193 F F F 4 080-099 99 99 99 191 191 191 F F F 5 100-119 99 99 99 191 191 191 T T T 6 120-139 95 95 95 186 186 186 F F F 7 140-159 100 100 100 193 193 193 F F F 8 160-179 100 100 100 193 193 193 F F F 9 180-199 92 92 92 181 181 181 F F F 10 200-219 100 100 100 193 193 193 T T T 11 220-239 92 92 92 181 181 181 F F F 12 240-259 98 98 98 190 190 190 F F F 13 260-279 97 97 97 188 188 188 F F F 14 280-299 99 99 99 191 191 191 F F F 15 300-319 96 96 96 187 187 187 F F F 16 320-339 89 89 89 173 173 173 F F F 17 340-359 95 95 95 186 186 186 F F F 18 360-379 100 100 100 193 193 193 T T T 19 380-399 99 99 99 191 191 191 T T T 20 400-419 99 99 99 191 191 191 F F F 21 420-439 100 100 100 192 192 192 F F F 22 440-459 100 100 100 192 192 192 F F F 23 460-479 90 90 90 177 177 177 F F F 24 480-499 100 100 100 193 193 193 F F F 25 500-519 100 100 100 193 193 193 F F F 26 520-539 98 98 98 189 189 189 F F F 27 540-559 98 98 98 189 189 189 T T T 28 560-579 99 99 99 191 191 191 F F F 29 580-599 97 97 97 188 188 188 F F F Max gen: 100, Max flush: 193 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 194 194 194 F F F 1 020-039 100 100 100 193 193 193 F F F 2 040-059 100 100 100 193 193 193 F F F 3 060-079 100 100 100 193 193 193 F F F 4 080-099 99 99 99 191 191 191 F F F 5 100-119 101 101 101 194 194 194 F F F 6 120-139 95 95 95 186 186 186 F F F 7 140-159 100 100 100 193 193 193 F F F 8 160-179 100 100 100 193 193 193 F F F 9 180-199 92 92 92 181 181 181 F F F 10 200-219 101 101 101 194 194 194 F F F 11 220-239 92 92 92 181 181 181 F F F 12 240-259 98 98 98 190 190 190 F F F 13 260-279 97 97 97 188 188 188 F F F 14 280-299 99 99 99 191 191 191 F F F 15 300-319 96 96 96 187 187 187 F F F 16 320-339 89 89 89 173 173 173 F F F 17 340-359 95 95 95 186 186 186 F F F 18 360-379 101 101 101 194 194 194 F F F 19 380-399 101 101 101 194 194 194 F F F 20 400-419 99 99 99 191 191 191 F F F 21 420-439 100 100 100 192 192 192 F F F 22 440-459 100 100 100 192 192 192 F F F 23 460-479 90 90 90 177 177 177 F F F 24 480-499 100 100 100 193 193 193 F F F 25 500-519 100 100 100 193 193 193 F F F 26 520-539 98 98 98 189 189 189 F F F 27 540-559 101 101 101 194 194 194 F F F 28 560-579 99 99 99 191 191 191 F F F 29 580-599 97 97 97 188 188 188 F F F Max gen: 101, Max flush: 194 Begin loop 51 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.551675078Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.596415789Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.597058753Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.597095334Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.597109354Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.597123754Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595} {"msg":"Crucible 7f0663be-206e-4be4-ac12-e17576d56163 has session id: 416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.597593394Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"Upstairs opts: Upstairs UUID: 7f0663be-206e-4be4-ac12-e17576d56163, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.597638005Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"Crucible stats registered with UUID: 7f0663be-206e-4be4-ac12-e17576d56163","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.597655656Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.62917291Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.62920457Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595} {"msg":"7f0663be-206e-4be4-ac12-e17576d56163 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.629253821Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.629274352Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"client":"0","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.629309482Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"client":"0","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.629342703Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"client":"1","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.629374984Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"client":"1","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.629403915Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"client":"2","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.629430675Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"client":"2","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.629456985Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.629483476Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.629510357Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:38064)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.629549167Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:41573)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.629576188Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:57176)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.629602649Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.630331275Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"client":"0","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.630376296Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"client":"0","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.630405506Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"client":"2","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.630434807Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"client":"2","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.630527809Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"client":"1","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.63056728Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"client":"1","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.630998549Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"client":"0","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.63102878Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"client":"2","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.631166313Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"client":"1","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.631532511Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"client":"2","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.631574391Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"client":"2","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.631604412Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.631675464Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"client":"1","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.631703604Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"client":"1","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.631719145Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.631732285Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"client":"0","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.631746005Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"client":"0","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"[0]R flush_numbers[0..12]: [194, 193, 193, 193, 191, 194, 186, 193, 193, 181, 194, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.631760115Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"":"downstairs","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"[0]R generation[0..12]: [101, 100, 100, 100, 99, 101, 95, 100, 100, 92, 101, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.631774356Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"":"downstairs","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.631792176Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"":"downstairs","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"[1]R flush_numbers[0..12]: [194, 193, 193, 193, 191, 194, 186, 193, 193, 181, 194, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.631806627Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"":"downstairs","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"[1]R generation[0..12]: [101, 100, 100, 100, 99, 101, 95, 100, 100, 92, 101, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.631820857Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"":"downstairs","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.631844247Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"":"downstairs","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"[2]R flush_numbers[0..12]: [194, 193, 193, 193, 191, 194, 186, 193, 193, 181, 194, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.631873588Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"":"downstairs","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"[2]R generation[0..12]: [101, 100, 100, 100, 99, 101, 95, 100, 100, 92, 101, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.631896449Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"":"downstairs","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:53:52.631921499Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"":"downstairs","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.63195178Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"":"downstairs","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.63197836Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"":"downstairs","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"Next flush: 195","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.631994061Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"":"downstairs","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.632018321Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"":"downstairs","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.632044172Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.632070962Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"client":"0","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.632091283Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"client":"1","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.632105933Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"client":"2","":"downstairs","session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.632134634Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.632161274Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"7f0663be-206e-4be4-ac12-e17576d56163 is now active with session: 416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.632196985Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595,"session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.632220616Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 1595,"session_id":"416f25b7-5a4e-447d-8d7e-87d6d5a3a1e5"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:52.632246936Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 101 len 32768 data: 8 9 9 8 6 5 5 5 02/30 Read block 496 len 16384 03/30 Read block 110 len 36864 04/30 Read block 511 len 20480 05/30 Read block 64 len 16384 06/30 Read block 140 len 36864 07/30 Read block 593 len 8192 08/30 Write block 30 len 28672 data: 5 6 6 5 5 7 6 09/30 Read block 384 len 8192 10/30 Read block 349 len 40960 11/30 Flush 12/30 Write block 523 len 36864 data: 9 12 11 13 10 9 9 7 6 13/30 Write block 45 len 8192 data: 10 10 14/30 Read block 186 len 8192 15/30 Read block 574 len 12288 16/30 Read block 168 len 28672 17/30 Write block 294 len 40960 data: 5 4 5 5 7 8 8 8 8 7 18/30 Read block 209 len 36864 19/30 Read block 413 len 36864 20/30 Read block 5 len 40960 21/30 Write block 376 len 24576 data: 8 10 9 9 8 8 22/30 Read block 286 len 28672 23/30 Write block 537 len 40960 data: 7 8 8 6 4 6 6 7 12 11 24/30 Read block 61 len 8192 25/30 Read block 28 len 40960 26/30 Write block 8 len 20480 data: 7 6 4 6 6 27/30 Write block 194 len 28672 data: 12 14 10 13 12 10 11 28/30 Write block 504 len 36864 data: 11 9 11 12 10 11 10 9 10 29/30 Write block 527 len 16384 data: 11 10 10 8 30/30 Read block 6 len 36864 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 9 Done Done Done false 1012 Acked Write 2 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 4 Sent Done Sent false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 2 4 Done 17 19 17 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:53:52.653211941Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1595} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 194 194 194 T T T 1 020-039 102 102 102 195 195 195 F F F 2 040-059 100 100 100 193 193 193 T T T 3 060-079 100 100 100 193 193 193 F F F 4 080-099 99 99 99 191 191 191 F F F 5 100-119 102 102 102 195 195 195 F F F 6 120-139 95 95 95 186 186 186 F F F 7 140-159 100 100 100 193 193 193 F F F 8 160-179 100 100 100 193 193 193 F F F 9 180-199 92 92 92 181 181 181 T T T 10 200-219 101 101 101 194 194 194 T T T 11 220-239 92 92 92 181 181 181 F F F 12 240-259 98 98 98 190 190 190 F F F 13 260-279 97 97 97 188 188 188 F F F 14 280-299 99 99 99 191 191 191 T T T 15 300-319 96 96 96 187 187 187 T T T 16 320-339 89 89 89 173 173 173 F F F 17 340-359 95 95 95 186 186 186 F F F 18 360-379 101 101 101 194 194 194 T T T 19 380-399 101 101 101 194 194 194 T T T 20 400-419 99 99 99 191 191 191 F F F 21 420-439 100 100 100 192 192 192 F F F 22 440-459 100 100 100 192 192 192 F F F 23 460-479 90 90 90 177 177 177 F F F 24 480-499 100 100 100 193 193 193 F F F 25 500-519 100 100 100 193 193 193 T T T 26 520-539 98 98 98 189 189 189 T T T 27 540-559 101 101 101 194 194 194 T T T 28 560-579 99 99 99 191 191 191 F F F 29 580-599 97 97 97 188 188 188 F F F Max gen: 102, Max flush: 195 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 103 103 103 196 196 196 F F F 1 020-039 102 102 102 195 195 195 F F F 2 040-059 103 103 103 196 196 196 F F F 3 060-079 100 100 100 193 193 193 F F F 4 080-099 99 99 99 191 191 191 F F F 5 100-119 102 102 102 195 195 195 F F F 6 120-139 95 95 95 186 186 186 F F F 7 140-159 100 100 100 193 193 193 F F F 8 160-179 100 100 100 193 193 193 F F F 9 180-199 103 103 103 196 196 196 F F F 10 200-219 103 103 103 196 196 196 F F F 11 220-239 92 92 92 181 181 181 F F F 12 240-259 98 98 98 190 190 190 F F F 13 260-279 97 97 97 188 188 188 F F F 14 280-299 103 103 103 196 196 196 F F F 15 300-319 103 103 103 196 196 196 F F F 16 320-339 89 89 89 173 173 173 F F F 17 340-359 95 95 95 186 186 186 F F F 18 360-379 103 103 103 196 196 196 F F F 19 380-399 103 103 103 196 196 196 F F F 20 400-419 99 99 99 191 191 191 F F F 21 420-439 100 100 100 192 192 192 F F F 22 440-459 100 100 100 192 192 192 F F F 23 460-479 90 90 90 177 177 177 F F F 24 480-499 100 100 100 193 193 193 F F F 25 500-519 103 103 103 196 196 196 F F F 26 520-539 103 103 103 196 196 196 F F F 27 540-559 103 103 103 196 196 196 F F F 28 560-579 99 99 99 191 191 191 F F F 29 580-599 97 97 97 188 188 188 F F F Max gen: 103, Max flush: 196 Begin loop 52 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:56.959139819Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.003243286Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.003877579Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.00391389Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.003929811Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.003944771Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603} {"msg":"Crucible 5054bbbf-5254-4d84-8b1b-fee5a8a631a0 has session id: 925c3cdc-555d-4d47-9f8b-7737e250b352","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.004453732Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"Upstairs opts: Upstairs UUID: 5054bbbf-5254-4d84-8b1b-fee5a8a631a0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.004501123Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"Crucible stats registered with UUID: 5054bbbf-5254-4d84-8b1b-fee5a8a631a0","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.004526424Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.035309311Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.035345702Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603} {"msg":"5054bbbf-5254-4d84-8b1b-fee5a8a631a0 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.035405643Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.035481285Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"client":"0","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.035526526Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"client":"0","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.035558037Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"client":"1","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.035590147Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"client":"1","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.035606258Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"client":"2","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.035620138Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"client":"2","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.035633338Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.035647909Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.035671539Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"ds_connection connected from Ok(127.0.0.1:63258)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.03570147Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"ds_connection connected from Ok(127.0.0.1:35336)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.035902174Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"ds_connection connected from Ok(127.0.0.1:57841)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.035949135Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.036324653Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"client":"1","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.036352854Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"client":"1","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.036481367Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"client":"0","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.036526018Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"client":"0","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.036570039Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"client":"2","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.036590829Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"client":"2","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.036930356Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"client":"1","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.036955937Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"client":"0","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.037251814Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"client":"2","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.037482988Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"client":"0","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.037509249Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"client":"0","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.037524329Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.037607091Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"client":"1","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.037628952Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"client":"1","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.037642992Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.037681013Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"client":"2","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.037697953Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"client":"2","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"[0]R flush_numbers[0..12]: [196, 195, 196, 193, 191, 195, 186, 193, 193, 196, 196, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.037715844Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"":"downstairs","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"[0]R generation[0..12]: [103, 102, 103, 100, 99, 102, 95, 100, 100, 103, 103, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.037729864Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"":"downstairs","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.037744294Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"":"downstairs","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"[1]R flush_numbers[0..12]: [196, 195, 196, 193, 191, 195, 186, 193, 193, 196, 196, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.037762434Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"":"downstairs","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"[1]R generation[0..12]: [103, 102, 103, 100, 99, 102, 95, 100, 100, 103, 103, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.037776715Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"":"downstairs","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.037790905Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"":"downstairs","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"[2]R flush_numbers[0..12]: [196, 195, 196, 193, 191, 195, 186, 193, 193, 196, 196, 181]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.037804835Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"":"downstairs","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"[2]R generation[0..12]: [103, 102, 103, 100, 99, 102, 95, 100, 100, 103, 103, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.037818596Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"":"downstairs","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.037832626Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"":"downstairs","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.037845656Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"":"downstairs","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.037858977Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"":"downstairs","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"Next flush: 197","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.037872207Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"":"downstairs","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.037885397Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":Wait for a query_work_queue command to finish before sending IO 1603,"":"downstairs","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.037913298Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.037938108Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"client":"0","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.037964129Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"client":"1","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.03798905Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"client":"2","":"downstairs","session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-28T04:53:57.03801884Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.038037031Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"5054bbbf-5254-4d84-8b1b-fee5a8a631a0 is now active with session: 925c3cdc-555d-4d47-9f8b-7737e250b352","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.038050881Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.038063181Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603,"session_id":"925c3cdc-555d-4d47-9f8b-7737e250b352"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:53:57.038075761Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 400 len 16384 data: 9 10 9 8 02/30 Write block 429 len 32768 data: 9 8 9 8 9 9 9 9 03/30 Flush 04/30 Flush 05/30 Write block 227 len 40960 data: 7 5 6 6 8 10 11 12 10 11 06/30 Write block 490 len 36864 data: 8 8 8 6 8 10 10 10 12 07/30 Read block 419 len 16384 08/30 Read block 21 len 4096 09/30 Read block 92 len 16384 10/30 Write block 284 len 12288 data: 8 10 10 11/30 Read block 420 len 4096 12/30 Read block 207 len 16384 13/30 Read block 192 len 36864 14/30 Read block 366 len 36864 15/30 Write block 286 len 20480 data: 11 9 7 7 7 16/30 Read block 416 len 4096 17/30 Read block 554 len 24576 18/30 Read block 403 len 16384 19/30 Read block 372 len 8192 20/30 Read block 491 len 12288 21/30 Write block 593 len 4096 data: 6 22/30 Read block 340 len 24576 23/30 Read block 156 len 12288 24/30 Read block 106 len 36864 25/30 Read block 498 len 8192 26/30 Read block 174 len 32768 27/30 Write block 519 len 28672 data: 8 8 9 7 10 13 12 28/30 Write block 475 len 28672 data: 9 11 10 10 8 9 8 29/30 Write block 390 len 32768 data: 5 6 8 8 6 7 7 6 30/30 Read block 121 len 8192 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 10 Done Done Done false 1005 Acked Write 9 Done Done Done false 1006 Acked Read 4 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Read 4 Done Done Done false 1009 Acked Write 3 Done Done Done false 1010 Acked Read 1 Done Done Done false 1011 Acked Read 4 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Write 5 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 8 Done Sent Sent false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 26 24 24 74 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:53:57.057022742Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1603} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 103 103 103 196 196 196 F F F 1 020-039 102 102 102 195 195 195 F F F 2 040-059 103 103 103 196 196 196 F F F 3 060-079 100 100 100 193 193 193 F F F 4 080-099 99 99 99 191 191 191 F F F 5 100-119 102 102 102 195 195 195 F F F 6 120-139 95 95 95 186 186 186 F F F 7 140-159 100 100 100 193 193 193 F F F 8 160-179 100 100 100 193 193 193 F F F 9 180-199 103 103 103 196 196 196 F F F 10 200-219 103 103 103 196 196 196 F F F 11 220-239 92 92 92 181 181 181 T T T 12 240-259 98 98 98 190 190 190 F F F 13 260-279 97 97 97 188 188 188 F F F 14 280-299 103 103 103 196 196 196 T T T 15 300-319 103 103 103 196 196 196 F F F 16 320-339 89 89 89 173 173 173 F F F 17 340-359 95 95 95 186 186 186 F F F 18 360-379 103 103 103 196 196 196 F F F 19 380-399 103 103 103 196 196 196 T T T 20 400-419 104 104 104 197 197 197 F F F 21 420-439 104 104 104 197 197 197 F F F 22 440-459 100 100 100 192 192 192 F F F 23 460-479 90 90 90 177 177 177 T T T 24 480-499 100 100 100 193 193 193 T T T 25 500-519 103 103 103 196 196 196 T T T 26 520-539 103 103 103 196 196 196 T T T 27 540-559 103 103 103 196 196 196 F F F 28 560-579 99 99 99 191 191 191 F F F 29 580-599 97 97 97 188 188 188 T T T Max gen: 104, Max flush: 197 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 103 103 103 196 196 196 F F F 1 020-039 102 102 102 195 195 195 F F F 2 040-059 103 103 103 196 196 196 F F F 3 060-079 100 100 100 193 193 193 F F F 4 080-099 99 99 99 191 191 191 F F F 5 100-119 102 102 102 195 195 195 F F F 6 120-139 95 95 95 186 186 186 F F F 7 140-159 100 100 100 193 193 193 F F F 8 160-179 100 100 100 193 193 193 F F F 9 180-199 103 103 103 196 196 196 F F F 10 200-219 103 103 103 196 196 196 F F F 11 220-239 105 105 105 198 198 198 F F F 12 240-259 98 98 98 190 190 190 F F F 13 260-279 97 97 97 188 188 188 F F F 14 280-299 105 105 105 198 198 198 F F F 15 300-319 103 103 103 196 196 196 F F F 16 320-339 89 89 89 173 173 173 F F F 17 340-359 95 95 95 186 186 186 F F F 18 360-379 103 103 103 196 196 196 F F F 19 380-399 105 105 105 198 198 198 F F F 20 400-419 104 104 104 197 197 197 F F F 21 420-439 104 104 104 197 197 197 F F F 22 440-459 100 100 100 192 192 192 F F F 23 460-479 105 105 105 198 198 198 F F F 24 480-499 105 105 105 198 198 198 F F F 25 500-519 105 105 105 198 198 198 F F F 26 520-539 105 105 105 198 198 198 F F F 27 540-559 103 103 103 196 196 196 F F F 28 560-579 99 99 99 191 191 191 F F F 29 580-599 105 105 105 198 198 198 F F F Max gen: 105, Max flush: 198 Begin loop 53 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.336107939Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.379728285Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.380354749Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.380390899Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.3804061Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.38042089Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611} {"msg":"Crucible 2d149334-74d2-4323-957f-c8c4ba55309b has session id: 6dcc0254-6a83-4a86-abd1-b51617dcf7e0","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.380910501Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"Upstairs opts: Upstairs UUID: 2d149334-74d2-4323-957f-c8c4ba55309b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.380956042Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"Crucible stats registered with UUID: 2d149334-74d2-4323-957f-c8c4ba55309b","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.380980252Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.412506266Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.412540327Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611} {"msg":"2d149334-74d2-4323-957f-c8c4ba55309b active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.412591738Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.412612398Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"client":"0","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.412642619Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"client":"0","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.412657199Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"client":"1","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.412670639Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"client":"1","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.41269419Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"client":"2","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.412723631Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"client":"2","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.412751981Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"":"io task","client":"2","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.412781322Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"":"io task","client":"0","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.412810323Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:55123)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.412838673Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"":"io task","client":"2","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:58932)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.412866964Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"":"io task","client":"0","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:36197)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.413075428Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.413656391Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"client":"0","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.413695062Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"client":"0","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.413717292Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"client":"1","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.413732743Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"client":"1","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.413814654Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"client":"2","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.413839895Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"client":"2","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.414310485Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"client":"0","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.414350346Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"client":"1","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.414639732Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"client":"2","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.414888218Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"client":"1","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.414916978Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"client":"1","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.414932369Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.414962839Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"client":"0","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.41497756Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"client":"0","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.41499123Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.415112603Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"client":"2","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.415131633Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"client":"2","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"[0]R flush_numbers[0..12]: [196, 195, 196, 193, 191, 195, 186, 193, 193, 196, 196, 198]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.415145813Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"":"downstairs","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"[0]R generation[0..12]: [103, 102, 103, 100, 99, 102, 95, 100, 100, 103, 103, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.415160073Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"":"downstairs","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.415178214Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"":"downstairs","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"[1]R flush_numbers[0..12]: [196, 195, 196, 193, 191, 195, 186, 193, 193, 196, 196, 198]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.415192944Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"":"downstairs","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"[1]R generation[0..12]: [103, 102, 103, 100, 99, 102, 95, 100, 100, 103, 103, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.415206935Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"":"downstairs","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.415220845Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"":"downstairs","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"[2]R flush_numbers[0..12]: [196, 195, 196, 193, 191, 195, 186, 193, 193, 196, 196, 198]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.415234845Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"":"downstairs","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"[2]R generation[0..12]: [103, 102, 103, 100, 99, 102, 95, 100, 100, 103, 103, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.415248506Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"":"downstairs","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.415262456Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"":"downstairs","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.415275426Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"":"downstairs","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.415288746Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"":"downstairs","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"Next flush: 199","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.415301807Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"":"downstairs","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.415314847Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid"Wait for a query_work_queue command to finish before sending IO :1611,"":"downstairs","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.415336808Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.415348968Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"client":"0","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.415362188Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"client":"1","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.415375288Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"client":"2","":"downstairs","session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.415390518Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.415403189Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"2d149334-74d2-4323-957f-c8c4ba55309b is now active with session: 6dcc0254-6a83-4a86-abd1-b51617dcf7e0","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.415416099Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.415428199Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611,"session_id":"6dcc0254-6a83-4a86-abd1-b51617dcf7e0"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:01.41544058Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 6 len 24576 data: 6 6 8 7 5 7 02/30 Read block 119 len 20480 03/30 Read block 482 len 28672 04/30 Read block 183 len 16384 05/30 Write block 169 len 24576 data: 8 8 9 9 8 9 06/30 Write block 443 len 8192 data: 9 7 07/30 Read block 270 len 32768 08/30 Read block 251 len 40960 09/30 Flush 10/30 Write block 105 len 24576 data: 7 6 6 6 6 8 11/30 Write block 387 len 12288 data: 8 8 8 12/30 Read block 465 len 8192 13/30 Write block 538 len 8192 data: 9 9 14/30 Read block 288 len 36864 15/30 Read block 534 len 12288 16/30 Read block 441 len 16384 17/30 Read block 552 len 12288 18/30 Read block 43 len 32768 19/30 Flush 20/30 Write block 594 len 16384 data: 5 4 4 5 21/30 Flush 22/30 Write block 592 len 4096 data: 8 23/30 Write block 50 len 4096 data: 11 24/30 Read block 378 len 40960 25/30 Flush 26/30 Flush 27/30 Write block 430 len 16384 data: 9 10 9 10 28/30 Write block 62 len 40960 data: 8 7 7 8 8 10 11 11 10 14 29/30 Write block 525 len 32768 data: 13 14 12 11 11 9 7 6 30/30 Write block 169 len 4096 data: 9 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 4 Done Done Done false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 1 1 1 3 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:54:01.435581407Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1611} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 199 199 199 F F F 1 020-039 102 102 102 195 195 195 F F F 2 040-059 106 106 106 202 202 202 F F F 3 060-079 100 100 100 193 193 193 T T T 4 080-099 99 99 99 191 191 191 F F F 5 100-119 106 106 106 200 200 200 F F F 6 120-139 95 95 95 186 186 186 F F F 7 140-159 100 100 100 193 193 193 F F F 8 160-179 106 106 106 199 199 199 T T T 9 180-199 103 103 103 196 196 196 F F F 10 200-219 103 103 103 196 196 196 F F F 11 220-239 105 105 105 198 198 198 F F F 12 240-259 98 98 98 190 190 190 F F F 13 260-279 97 97 97 188 188 188 F F F 14 280-299 105 105 105 198 198 198 F F F 15 300-319 103 103 103 196 196 196 F F F 16 320-339 89 89 89 173 173 173 F F F 17 340-359 95 95 95 186 186 186 F F F 18 360-379 103 103 103 196 196 196 F F F 19 380-399 106 106 106 200 200 200 F F F 20 400-419 104 104 104 197 197 197 F F F 21 420-439 104 104 104 197 197 197 T T T 22 440-459 106 106 106 199 199 199 F F F 23 460-479 105 105 105 198 198 198 F F F 24 480-499 105 105 105 198 198 198 F F F 25 500-519 105 105 105 198 198 198 F F F 26 520-539 106 106 106 200 200 200 T T T 27 540-559 103 103 103 196 196 196 F F F 28 560-579 99 99 99 191 191 191 F F F 29 580-599 106 106 106 202 202 202 F F F Max gen: 106, Max flush: 202 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 199 199 199 F F F 1 020-039 102 102 102 195 195 195 F F F 2 040-059 106 106 106 202 202 202 F F F 3 060-079 107 107 107 203 203 203 F F F 4 080-099 99 99 99 191 191 191 F F F 5 100-119 106 106 106 200 200 200 F F F 6 120-139 95 95 95 186 186 186 F F F 7 140-159 100 100 100 193 193 193 F F F 8 160-179 107 107 107 203 203 203 F F F 9 180-199 103 103 103 196 196 196 F F F 10 200-219 103 103 103 196 196 196 F F F 11 220-239 105 105 105 198 198 198 F F F 12 240-259 98 98 98 190 190 190 F F F 13 260-279 97 97 97 188 188 188 F F F 14 280-299 105 105 105 198 198 198 F F F 15 300-319 103 103 103 196 196 196 F F F 16 320-339 89 89 89 173 173 173 F F F 17 340-359 95 95 95 186 186 186 F F F 18 360-379 103 103 103 196 196 196 F F F 19 380-399 106 106 106 200 200 200 F F F 20 400-419 104 104 104 197 197 197 F F F 21 420-439 107 107 107 203 203 203 F F F 22 440-459 106 106 106 199 199 199 F F F 23 460-479 105 105 105 198 198 198 F F F 24 480-499 105 105 105 198 198 198 F F F 25 500-519 105 105 105 198 198 198 F F F 26 520-539 107 107 107 203 203 203 F F F 27 540-559 103 103 103 196 196 196 F F F 28 560-579 99 99 99 191 191 191 F F F 29 580-599 106 106 106 202 202 202 F F F Max gen: 107, Max flush: 203 Begin loop 54 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.700818542Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:54:05.744841557Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.745222756Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.778110409Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.778652801Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.778690471Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.778705462Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.778721032Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619} {"msg":"Crucible b7a85efc-a6f1-40da-9950-7ce92705f093 has session id: af82b3cf-e57a-4ecb-9634-a527894f0494","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.779142541Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"Upstairs opts: Upstairs UUID: b7a85efc-a6f1-40da-9950-7ce92705f093, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.779186062Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"Crucible stats registered with UUID: b7a85efc-a6f1-40da-9950-7ce92705f093","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.779210403Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.80996526Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.80999327Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619} {"msg":"b7a85efc-a6f1-40da-9950-7ce92705f093 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.810049902Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.810078462Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"client":"0","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.810104293Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"client":"0","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.810120123Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"client":"1","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.810134253Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"client":"1","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.810195875Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"client":"2","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.810233246Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"client":"2","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.810261536Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"":"io task","client":"2","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.810289617Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"":"io task","client":"0","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.810318397Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"ds_connection connected from Ok(127.0.0.1:35862)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.810359898Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"":"io task","client":"2","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"ds_connection connected from Ok(127.0.0.1:57553)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.81045538Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"ds_connection connected from Ok(127.0.0.1:32887)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.810483961Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"":"io task","client":"0","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.81091531Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"client":"1","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.810943711Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"client":"1","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.811081234Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"client":"2","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.811102314Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"client":"2","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.811306019Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"client":"0","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.811337439Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"client":"0","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.811573955Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"client":"1","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.811624606Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"client":"2","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.811941693Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"client":"0","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.812196608Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"client":"1","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.812228679Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"client":"1","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.81225557Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.81227198Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"client":"2","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.8122861Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"client":"2","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.812299991Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.812408573Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"client":"0","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.812427583Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"client":"0","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"[0]R flush_numbers[0..12]: [199, 195, 202, 203, 191, 200, 186, 193, 203, 196, 196, 198]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.812441954Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"":"downstairs","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"[0]R generation[0..12]: [106, 102, 106, 107, 99, 106, 95, 100, 107, 103, 103, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.812456224Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"":"downstairs","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.812470934Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"":"downstairs","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"[1]R flush_numbers[0..12]: [199, 195, 202, 203, 191, 200, 186, 193, 203, 196, 196, 198]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.812485055Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"":"downstairs","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"[1]R generation[0..12]: [106, 102, 106, 107, 99, 106, 95, 100, 107, 103, 103, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.812499085Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"":"downstairs","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.812516445Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"":"downstairs","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"[2]R flush_numbers[0..12]: [199, 195, 202, 203, 191, 200, 186, 193, 203, 196, 196, 198]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.812530816Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"":"downstairs","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"[2]R generation[0..12]: [106, 102, 106, 107, 99, 106, 95, 100, 107, 103, 103, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.812544726Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"":"downstairs","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.812558996Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"":"downstairs","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.812572056Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"":"downstairs","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.812585347Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"":"downstairs","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"Next flush: 204","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.812598337Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN"Wait for a query_work_queue command to finish before sending IO ,"pid":1619,"":"downstairs","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.812620467Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"":"downstairs","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.812633888Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.812646188Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"client":"0","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.812659558Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"client":"1","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.812672679Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"client":"2","":"downstairs","session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.812686089Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.812698539Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"b7a85efc-a6f1-40da-9950-7ce92705f093 is now active with session: af82b3cf-e57a-4ecb-9634-a527894f0494","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.812711519Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.81272646Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619,"session_id":"af82b3cf-e57a-4ecb-9634-a527894f0494"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:05.81273964Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 570 len 24576 data: 10 11 11 13 14 14 02/30 Read block 222 len 40960 03/30 Read block 129 len 4096 04/30 Read block 180 len 40960 05/30 Write block 98 len 4096 data: 7 06/30 Read block 89 len 8192 07/30 Write block 527 len 16384 data: 13 12 12 10 08/30 Read block 161 len 40960 09/30 Write block 65 len 12288 data: 9 9 11 10/30 Read block 593 len 4096 11/30 Write block 175 len 28672 data: 9 9 7 5 5 4 4 12/30 Read block 48 len 24576 13/30 Write block 480 len 20480 data: 10 9 9 8 9 14/30 Read block 310 len 36864 15/30 Read block 234 len 12288 16/30 Write block 188 len 20480 data: 11 12 11 12 11 17/30 Read block 284 len 16384 18/30 Write block 74 len 28672 data: 16 14 11 10 10 9 6 19/30 Write block 314 len 4096 data: 8 20/30 Flush 21/30 Write block 75 len 32768 data: 15 12 11 11 10 7 8 6 22/30 Flush 23/30 Write block 265 len 24576 data: 8 10 10 10 12 15 24/30 Read block 566 len 12288 25/30 Write block 157 len 32768 data: 9 9 7 7 7 7 7 7 26/30 Read block 14 len 20480 27/30 Write block 366 len 32768 data: 7 7 7 8 8 7 7 6 28/30 Write block 409 len 16384 data: 8 11 9 8 29/30 Write block 128 len 12288 data: 8 6 7 30/30 Read block 53 len 4096 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 6 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:54:05.832321935Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1619} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 199 199 199 F F F 1 020-039 102 102 102 195 195 195 F F F 2 040-059 106 106 106 202 202 202 F F F 3 060-079 108 108 108 205 205 205 F F F 4 080-099 108 108 108 205 205 205 F F F 5 100-119 106 106 106 200 200 200 F F F 6 120-139 95 95 95 186 186 186 T T T 7 140-159 100 100 100 193 193 193 T T T 8 160-179 108 108 108 204 204 204 T T T 9 180-199 108 108 108 204 204 204 F F F 10 200-219 103 103 103 196 196 196 F F F 11 220-239 105 105 105 198 198 198 F F F 12 240-259 98 98 98 190 190 190 F F F 13 260-279 97 97 97 188 188 188 T T T 14 280-299 105 105 105 198 198 198 F F F 15 300-319 108 108 108 204 204 204 F F F 16 320-339 89 89 89 173 173 173 F F F 17 340-359 95 95 95 186 186 186 F F F 18 360-379 103 103 103 196 196 196 T T T 19 380-399 106 106 106 200 200 200 F F F 20 400-419 104 104 104 197 197 197 T T T 21 420-439 107 107 107 203 203 203 F F F 22 440-459 106 106 106 199 199 199 F F F 23 460-479 105 105 105 198 198 198 F F F 24 480-499 108 108 108 204 204 204 F F F 25 500-519 105 105 105 198 198 198 F F F 26 520-539 108 108 108 204 204 204 F F F 27 540-559 103 103 103 196 196 196 F F F 28 560-579 108 108 108 204 204 204 F F F 29 580-599 106 106 106 202 202 202 F F F Max gen: 108, Max flush: 205 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 199 199 199 F F F 1 020-039 102 102 102 195 195 195 F F F 2 040-059 106 106 106 202 202 202 F F F 3 060-079 108 108 108 205 205 205 F F F 4 080-099 108 108 108 205 205 205 F F F 5 100-119 106 106 106 200 200 200 F F F 6 120-139 109 109 109 206 206 206 F F F 7 140-159 109 109 109 206 206 206 F F F 8 160-179 109 109 109 206 206 206 F F F 9 180-199 108 108 108 204 204 204 F F F 10 200-219 103 103 103 196 196 196 F F F 11 220-239 105 105 105 198 198 198 F F F 12 240-259 98 98 98 190 190 190 F F F 13 260-279 109 109 109 206 206 206 F F F 14 280-299 105 105 105 198 198 198 F F F 15 300-319 108 108 108 204 204 204 F F F 16 320-339 89 89 89 173 173 173 F F F 17 340-359 95 95 95 186 186 186 F F F 18 360-379 109 109 109 206 206 206 F F F 19 380-399 106 106 106 200 200 200 F F F 20 400-419 109 109 109 206 206 206 F F F 21 420-439 107 107 107 203 203 203 F F F 22 440-459 106 106 106 199 199 199 F F F 23 460-479 105 105 105 198 198 198 F F F 24 480-499 108 108 108 204 204 204 F F F 25 500-519 105 105 105 198 198 198 F F F 26 520-539 108 108 108 204 204 204 F F F 27 540-559 103 103 103 196 196 196 F F F 28 560-579 108 108 108 204 204 204 F F F 29 580-599 106 106 106 202 202 202 F F F Max gen: 109, Max flush: 206 Begin loop 55 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.136596527Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.180307896Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.181162714Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.181199035Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.181218786Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.181235026Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627} {"msg":"Crucible 172ad031-ff4e-4296-b16c-6b9f1fd7677d has session id: 4b98f0a2-6a00-4998-95f6-089f0714863b","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.181690356Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"Upstairs opts: Upstairs UUID: 172ad031-ff4e-4296-b16c-6b9f1fd7677d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.181732997Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"Crucible stats registered with UUID: 172ad031-ff4e-4296-b16c-6b9f1fd7677d","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.181750787Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.21274882Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.21278314Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627} {"msg":"172ad031-ff4e-4296-b16c-6b9f1fd7677d active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.212843082Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.212872012Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"client":"0","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.212898623Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"client":"0","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.212913603Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"client":"1","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.212928623Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"client":"1","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.212942314Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"client":"2","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.212959384Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"client":"2","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.212985565Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"":"io task","client":"2","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.213020245Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"":"io task","client":"0","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.213053356Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"":"io task","client":"1","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"ds_connection connected from Ok(127.0.0.1:52258)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.213080967Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"":"io task","client":"2","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"ds_connection connected from Ok(127.0.0.1:54165)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.213108277Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"":"io task","client":"0","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"ds_connection connected from Ok(127.0.0.1:44636)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.213257841Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"":"io task","client":"1","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.213750981Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"client":"2","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.213779332Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"client":"2","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.213795022Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"client":"0","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.213809213Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"client":"0","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.213953826Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"client":"1","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.213999637Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"client":"1","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.214425126Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"client":"0","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.214470377Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"client":"1","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.214559279Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"client":"2","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.214965958Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"client":"0","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.214995728Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"client":"0","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.215011249Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.215102251Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"client":"1","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.215122481Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"client":"1","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.215136451Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.215148972Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"client":"2","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.215162372Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"client":"2","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"[0]R flush_numbers[0..12]: [199, 195, 202, 205, 205, 200, 206, 206, 206, 204, 196, 198]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.215180162Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"":"downstairs","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"[0]R generation[0..12]: [106, 102, 106, 108, 108, 106, 109, 109, 109, 108, 103, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.215194143Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"":"downstairs","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.215208433Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"":"downstairs","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"[1]R flush_numbers[0..12]: [199, 195, 202, 205, 205, 200, 206, 206, 206, 204, 196, 198]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.215222573Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"":"downstairs","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"[1]R generation[0..12]: [106, 102, 106, 108, 108, 106, 109, 109, 109, 108, 103, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.215236364Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"":"downstairs","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.215250334Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"":"downstairs","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"[2]R flush_numbers[0..12]: [199, 195, 202, 205, 205, 200, 206, 206, 206, 204, 196, 198]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.215264184Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"":"downstairs","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"[2]R generation[0..12]: [106, 102, 106, 108, 108, 106, 109, 109, 109, 108, 103, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.215280974Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"":"downstairs","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.215295305Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"":"downstairs","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.215308445Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"":"downstairs","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.215321775Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"":"downstairs","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"Next flush: 207","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.215334706Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"":"downstairs","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.215347666Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"":"downstairs","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.215360986Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2024-11-28T04:54:10.215373316Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"client":"0","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.215395667Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"client":"1","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.215409187Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"client":"2","":"downstairs","session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.215422618Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.215435078Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"172ad031-ff4e-4296-b16c-6b9f1fd7677d is now active with session: 4b98f0a2-6a00-4998-95f6-089f0714863b","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.215448018Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.215460058Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627,"session_id":"4b98f0a2-6a00-4998-95f6-089f0714863b"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:10.215472399Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 116 len 24576 data: 9 9 8 7 5 7 02/30 Write block 514 len 20480 data: 9 10 9 10 10 03/30 Read block 543 len 28672 04/30 Read block 557 len 8192 05/30 Read block 538 len 32768 06/30 Read block 370 len 20480 07/30 Read block 523 len 36864 08/30 Read block 341 len 36864 09/30 Read block 527 len 32768 10/30 Write block 462 len 16384 data: 10 10 7 5 11/30 Write block 275 len 32768 data: 10 9 9 9 6 5 5 7 12/30 Read block 217 len 32768 13/30 Read block 139 len 36864 14/30 Read block 190 len 40960 15/30 Read block 10 len 24576 16/30 Read block 586 len 40960 17/30 Write block 53 len 28672 data: 11 13 13 14 14 14 12 18/30 Read block 386 len 24576 19/30 Read block 411 len 40960 20/30 Write block 21 len 28672 data: 11 9 11 10 9 9 9 21/30 Read block 496 len 8192 22/30 Read block 65 len 40960 23/30 Flush 24/30 Write block 560 len 20480 data: 7 4 4 4 2 25/30 Write block 538 len 36864 data: 10 10 7 5 7 7 8 13 12 26/30 Read block 266 len 40960 27/30 Write block 142 len 36864 data: 8 8 8 8 9 11 11 13 13 28/30 Write block 403 len 32768 data: 9 10 9 9 8 9 9 12 29/30 Write block 297 len 16384 data: 6 8 9 9 30/30 Write block 128 len 36864 data: 9 7 8 7 8 6 6 8 6 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 5 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:54:10.237228361Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1627} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 199 199 199 F F F 1 020-039 110 110 110 207 207 207 F F F 2 040-059 110 110 110 207 207 207 F F F 3 060-079 108 108 108 205 205 205 F F F 4 080-099 108 108 108 205 205 205 F F F 5 100-119 110 110 110 207 207 207 F F F 6 120-139 110 110 110 207 207 207 T T T 7 140-159 109 109 109 206 206 206 T T T 8 160-179 109 109 109 206 206 206 F F F 9 180-199 108 108 108 204 204 204 F F F 10 200-219 103 103 103 196 196 196 F F F 11 220-239 105 105 105 198 198 198 F F F 12 240-259 98 98 98 190 190 190 F F F 13 260-279 110 110 110 207 207 207 F F F 14 280-299 110 110 110 207 207 207 T T T 15 300-319 108 108 108 204 204 204 T T T 16 320-339 89 89 89 173 173 173 F F F 17 340-359 95 95 95 186 186 186 F F F 18 360-379 109 109 109 206 206 206 F F F 19 380-399 106 106 106 200 200 200 F F F 20 400-419 109 109 109 206 206 206 T T T 21 420-439 107 107 107 203 203 203 F F F 22 440-459 106 106 106 199 199 199 F F F 23 460-479 110 110 110 207 207 207 F F F 24 480-499 108 108 108 204 204 204 F F F 25 500-519 110 110 110 207 207 207 F F F 26 520-539 108 108 108 204 204 204 T T T 27 540-559 103 103 103 196 196 196 T T T 28 560-579 108 108 108 204 204 204 T T T 29 580-599 106 106 106 202 202 202 F F F Max gen: 110, Max flush: 207 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 199 199 199 F F F 1 020-039 110 110 110 207 207 207 F F F 2 040-059 110 110 110 207 207 207 F F F 3 060-079 108 108 108 205 205 205 F F F 4 080-099 108 108 108 205 205 205 F F F 5 100-119 110 110 110 207 207 207 F F F 6 120-139 111 111 111 208 208 208 F F F 7 140-159 111 111 111 208 208 208 F F F 8 160-179 109 109 109 206 206 206 F F F 9 180-199 108 108 108 204 204 204 F F F 10 200-219 103 103 103 196 196 196 F F F 11 220-239 105 105 105 198 198 198 F F F 12 240-259 98 98 98 190 190 190 F F F 13 260-279 110 110 110 207 207 207 F F F 14 280-299 111 111 111 208 208 208 F F F 15 300-319 111 111 111 208 208 208 F F F 16 320-339 89 89 89 173 173 173 F F F 17 340-359 95 95 95 186 186 186 F F F 18 360-379 109 109 109 206 206 206 F F F 19 380-399 106 106 106 200 200 200 F F F 20 400-419 111 111 111 208 208 208 F F F 21 420-439 107 107 107 203 203 203 F F F 22 440-459 106 106 106 199 199 199 F F F 23 460-479 110 110 110 207 207 207 F F F 24 480-499 108 108 108 204 204 204 F F F 25 500-519 110 110 110 207 207 207 F F F 26 520-539 111 111 111 208 208 208 F F F 27 540-559 111 111 111 208 208 208 F F F 28 560-579 111 111 111 208 208 208 F F F 29 580-599 106 106 106 202 202 202 F F F Max gen: 111, Max flush: 208 Begin loop 56 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.515705094Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:54:14.558518483Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.558892561Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.590411924Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.590956986Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.590993087Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.591008108Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.591027148Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635} {"msg":"Crucible accbd554-62c6-40d6-9860-72bef148952c has session id: b0c8e769-3daf-43e2-8212-ecec2e630d84","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.591496908Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"Upstairs opts: Upstairs UUID: accbd554-62c6-40d6-9860-72bef148952c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.591545769Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"Crucible stats registered with UUID: accbd554-62c6-40d6-9860-72bef148952c","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.59157127Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.622333377Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.622369158Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635} {"msg":"accbd554-62c6-40d6-9860-72bef148952c active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.622397778Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.622424169Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"client":"0","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.62247206Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"client":"0","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.622509051Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"client":"1","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.622545022Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"client":"1","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.622578532Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"client":"2","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.622627854Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"client":"2","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.622827938Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"":"io task","client":"2","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.622873099Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"":"io task","client":"0","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.622907189Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"":"io task","client":"1","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"ds_connection connected from Ok(127.0.0.1:62332)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.62293745Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"":"io task","client":"2","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"ds_connection connected from Ok(127.0.0.1:60081)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.622966111Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"":"io task","client":"1","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"ds_connection connected from Ok(127.0.0.1:56475)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.622994301Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"":"io task","client":"0","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.623217776Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"client":"2","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.623242867Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"client":"2","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.623365259Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"client":"1","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.62339094Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"client":"1","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.623691426Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"client":"0","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.623714557Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"client":"0","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.623743578Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"client":"2","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.623882171Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"client":"1","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.624332271Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"client":"2","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.624357961Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"client":"2","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.624372091Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.624447283Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"client":"1","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.624467323Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"client":"1","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.624480964Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.624494044Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"client":"0","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.624934824Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"client":"0","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.624953274Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"client":"0","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"[0]R flush_numbers[0..12]: [199, 207, 207, 205, 205, 207, 208, 208, 206, 204, 196, 198]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.624967444Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"":"downstairs","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"[0]R generation[0..12]: [106, 110, 110, 108, 108, 110, 111, 111, 109, 108, 103, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.624981585Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"":"downstairs","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.624996185Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"":"downstairs","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"[1]R flush_numbers[0..12]: [199, 207, 207, 205, 205, 207, 208, 208, 206, 204, 196, 198]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.625010225Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"":"downstairs","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"[1]R generation[0..12]: [106, 110, 110, 108, 108, 110, 111, 111, 109, 108, 103, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.625024415Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"":"downstairs","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.625038706Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"":"downstairs","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"[2]R flush_numbers[0..12]: [199, 207, 207, 205, 205, 207, 208, 208, 206, 204, 196, 198]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.625052786Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"":"downstairs","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"[2]R generation[0..12]: [106, 110, 110, 108, 108, 110, 111, 111, 109, 108, 103, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.625066476Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"":"downstairs","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.625083197Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"":"downstairs","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.625096657Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"":"downstairs","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.625110247Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"":"downstairs","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"Next flush: 209","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.625123337Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"":"downstairs","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.625136518Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"":"downstairs","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"Wait for a query_work_queue command to finish before sending IO No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.625158528Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.625170879Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"client":"0","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.625184159Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"client":"1","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.625197529Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"client":"2","":"downstairs","session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.625210839Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.62522327Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"accbd554-62c6-40d6-9860-72bef148952c is now active with session: b0c8e769-3daf-43e2-8212-ecec2e630d84","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.62523625Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.62524855Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635,"session_id":"b0c8e769-3daf-43e2-8212-ecec2e630d84"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:14.62526096Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 34 len 24576 data: 6 8 7 8 11 13 02/30 Read block 489 len 12288 03/30 Write block 8 len 16384 data: 9 8 6 8 04/30 Flush 05/30 Write block 466 len 28672 data: 5 6 7 8 7 7 5 06/30 Flush 07/30 Write block 168 len 8192 data: 8 10 08/30 Flush 09/30 Write block 541 len 32768 data: 6 8 8 9 14 13 11 11 10/30 Write block 390 len 8192 data: 6 7 11/30 Write block 512 len 4096 data: 11 12/30 Read block 358 len 12288 13/30 Read block 544 len 36864 14/30 Read block 316 len 8192 15/30 Flush 16/30 Write block 123 len 24576 data: 8 6 5 5 5 10 17/30 Write block 496 len 24576 data: 11 11 13 12 15 12 18/30 Read block 312 len 12288 19/30 Write block 581 len 16384 data: 6 5 8 8 20/30 Write block 211 len 36864 data: 11 11 10 11 11 11 8 9 8 21/30 Write block 473 len 24576 data: 5 9 10 12 11 11 22/30 Write block 387 len 16384 data: 9 9 9 7 23/30 Read block 69 len 24576 24/30 Write block 379 len 32768 data: 10 9 9 8 6 5 5 4 25/30 Read block 200 len 4096 26/30 Read block 477 len 12288 27/30 Read block 567 len 24576 28/30 Read block 90 len 24576 29/30 Read block 225 len 8192 30/30 Write block 107 len 36864 data: 7 7 7 9 9 10 10 11 11 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 6 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:54:14.643868014Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1635} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 209 209 209 F F F 1 020-039 112 112 112 209 209 209 F F F 2 040-059 110 110 110 207 207 207 F F F 3 060-079 108 108 108 205 205 205 F F F 4 080-099 108 108 108 205 205 205 F F F 5 100-119 110 110 110 207 207 207 T T T 6 120-139 111 111 111 208 208 208 T T T 7 140-159 111 111 111 208 208 208 F F F 8 160-179 112 112 112 211 211 211 F F F 9 180-199 108 108 108 204 204 204 F F F 10 200-219 103 103 103 196 196 196 T T T 11 220-239 105 105 105 198 198 198 F F F 12 240-259 98 98 98 190 190 190 F F F 13 260-279 110 110 110 207 207 207 F F F 14 280-299 111 111 111 208 208 208 F F F 15 300-319 111 111 111 208 208 208 F F F 16 320-339 89 89 89 173 173 173 F F F 17 340-359 95 95 95 186 186 186 F F F 18 360-379 109 109 109 206 206 206 T T T 19 380-399 112 112 112 212 212 212 T T T 20 400-419 111 111 111 208 208 208 F F F 21 420-439 107 107 107 203 203 203 F F F 22 440-459 106 106 106 199 199 199 F F F 23 460-479 112 112 112 210 210 210 T T T 24 480-499 108 108 108 204 204 204 T T T 25 500-519 112 112 112 212 212 212 T T T 26 520-539 111 111 111 208 208 208 F F F 27 540-559 112 112 112 212 212 212 F F F 28 560-579 111 111 111 208 208 208 F F F 29 580-599 106 106 106 202 202 202 T T T Max gen: 112, Max flush: 212 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 209 209 209 F F F 1 020-039 112 112 112 209 209 209 F F F 2 040-059 110 110 110 207 207 207 F F F 3 060-079 108 108 108 205 205 205 F F F 4 080-099 108 108 108 205 205 205 F F F 5 100-119 113 113 113 213 213 213 F F F 6 120-139 113 113 113 213 213 213 F F F 7 140-159 111 111 111 208 208 208 F F F 8 160-179 112 112 112 211 211 211 F F F 9 180-199 108 108 108 204 204 204 F F F 10 200-219 113 113 113 213 213 213 F F F 11 220-239 105 105 105 198 198 198 F F F 12 240-259 98 98 98 190 190 190 F F F 13 260-279 110 110 110 207 207 207 F F F 14 280-299 111 111 111 208 208 208 F F F 15 300-319 111 111 111 208 208 208 F F F 16 320-339 89 89 89 173 173 173 F F F 17 340-359 95 95 95 186 186 186 F F F 18 360-379 113 113 113 213 213 213 F F F 19 380-399 113 113 113 213 213 213 F F F 20 400-419 111 111 111 208 208 208 F F F 21 420-439 107 107 107 203 203 203 F F F 22 440-459 106 106 106 199 199 199 F F F 23 460-479 113 113 113 213 213 213 F F F 24 480-499 113 113 113 213 213 213 F F F 25 500-519 113 113 113 213 213 213 F F F 26 520-539 111 111 111 208 208 208 F F F 27 540-559 112 112 112 212 212 212 F F F 28 560-579 111 111 111 208 208 208 F F F 29 580-599 113 113 113 213 213 213 F F F Max gen: 113, Max flush: 213 Begin loop 57 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:18.966787931Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.010712544Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.011297237Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.011337618Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.011353368Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.011369578Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643} {"msg":"Crucible 84469c87-8e56-4fba-85cf-49ad8017735e has session id: 214152cd-dfe8-4970-a554-1929c0eee665","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.011771677Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"Upstairs opts: Upstairs UUID: 84469c87-8e56-4fba-85cf-49ad8017735e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.011816038Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"Crucible stats registered with UUID: 84469c87-8e56-4fba-85cf-49ad8017735e","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.011838178Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.042707328Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.042741379Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643} {"msg":"84469c87-8e56-4fba-85cf-49ad8017735e active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.042776509Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.04279477Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"client":"0","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.04282537Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"client":"0","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.042858071Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"client":"1","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.042892682Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"client":"1","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.042930923Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"client":"2","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.042965434Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"client":"2","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.043003225Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"":"io task","client":"2","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.043040765Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"":"io task","client":"0","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.043176698Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"":"io task","client":"1","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"ds_connection connected from Ok(127.0.0.1:65323)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.043215639Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"":"io task","client":"2","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"ds_connection connected from Ok(127.0.0.1:62120)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.04324748Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"":"io task","client":"0","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"ds_connection connected from Ok(127.0.0.1:43826)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.04327553Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"":"io task","client":"1","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.043759341Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"client":"0","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.043790522Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"client":"0","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.043817812Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"client":"1","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.043847223Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"client":"1","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.043915834Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"client":"2","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.043957835Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"client":"2","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.044372424Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"client":"1","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.044402835Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"client":"0","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.044493277Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"client":"2","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.044971957Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"client":"1","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.045000448Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"client":"1","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.045015588Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.045028278Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"client":"2","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.045044979Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"client":"2","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.045058609Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.045070879Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"client":"0","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.04508431Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"client":"0","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"[0]R flush_numbers[0..12]: [209, 209, 207, 205, 205, 213, 213, 208, 211, 204, 213, 198]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.0450982Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"":"downstairs","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"[0]R generation[0..12]: [112, 112, 110, 108, 108, 113, 113, 111, 112, 108, 113, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.04511241Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"":"downstairs","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.045126611Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"":"downstairs","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"[1]R flush_numbers[0..12]: [209, 209, 207, 205, 205, 213, 213, 208, 211, 204, 213, 198]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.045140701Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"":"downstairs","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"[1]R generation[0..12]: [112, 112, 110, 108, 108, 113, 113, 111, 112, 108, 113, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.045154561Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"":"downstairs","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.045168491Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"":"downstairs","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"[2]R flush_numbers[0..12]: [209, 209, 207, 205, 205, 213, 213, 208, 211, 204, 213, 198]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.045182302Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"":"downstairs","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"[2]R generation[0..12]: [112, 112, 110, 108, 108, 113, 113, 111, 112, 108, 113, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.045196872Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"":"downstairs","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.045210902Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"":"downstairs","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.045224203Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"":"downstairs","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.045240233Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"Next flush: 214","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.045262583Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"":"downstairs","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.045276264Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"":"downstairs","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.045289924Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.045302424Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"client":"0","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.045316095Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"client":"1","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.045329615Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"client":"2","":"downstairs","session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.045343325Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.045355816Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"84469c87-8e56-4fba-85cf-49ad8017735e is now active with session: 214152cd-dfe8-4970-a554-1929c0eee665","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.045368806Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 time":"2024-11-28T04:54:19.045381126Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643,"session_id":"214152cd-dfe8-4970-a554-1929c0eee665"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:19.045397816Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 547 len 12288 data: 12 12 10 02/30 Read block 346 len 40960 03/30 Read block 566 len 4096 04/30 Read block 325 len 40960 05/30 Write block 509 len 36864 data: 12 11 10 12 10 10 11 10 11 06/30 Write block 7 len 20480 data: 7 10 9 7 9 07/30 Read block 509 len 32768 08/30 Read block 459 len 20480 09/30 Write block 487 len 32768 data: 9 7 6 9 9 9 7 9 10/30 Read block 313 len 16384 11/30 Read block 568 len 20480 12/30 Read block 259 len 32768 13/30 Write block 203 len 28672 data: 10 8 10 10 7 8 6 14/30 Read block 28 len 8192 15/30 Write block 454 len 12288 data: 12 11 12 16/30 Read block 513 len 20480 17/30 Read block 47 len 8192 18/30 Read block 330 len 16384 19/30 Flush 20/30 Write block 284 len 4096 data: 9 21/30 Read block 567 len 20480 22/30 Read block 393 len 24576 23/30 Write block 18 len 20480 data: 9 12 10 12 10 24/30 Write block 508 len 36864 data: 11 13 12 11 13 11 11 12 11 25/30 Write block 479 len 4096 data: 9 26/30 Read block 15 len 4096 27/30 Read block 582 len 32768 28/30 Write block 507 len 12288 data: 13 12 14 29/30 Write block 496 len 24576 data: 12 12 14 13 16 13 30/30 Read block 397 len 28672 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 1 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 6 Done Done Sent false 1029 Acked Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 10 11 9 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:54:19.064688605Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1643} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 214 214 214 T T T 1 020-039 112 112 112 209 209 209 T T T 2 040-059 110 110 110 207 207 207 F F F 3 060-079 108 108 108 205 205 205 F F F 4 080-099 108 108 108 205 205 205 F F F 5 100-119 113 113 113 213 213 213 F F F 6 120-139 113 113 113 213 213 213 F F F 7 140-159 111 111 111 208 208 208 F F F 8 160-179 112 112 112 211 211 211 F F F 9 180-199 108 108 108 204 204 204 F F F 10 200-219 114 114 114 214 214 214 F F F 11 220-239 105 105 105 198 198 198 F F F 12 240-259 98 98 98 190 190 190 F F F 13 260-279 110 110 110 207 207 207 F F F 14 280-299 111 111 111 208 208 208 T T T 15 300-319 111 111 111 208 208 208 F F F 16 320-339 89 89 89 173 173 173 F F F 17 340-359 95 95 95 186 186 186 F F F 18 360-379 113 113 113 213 213 213 F F F 19 380-399 113 113 113 213 213 213 F F F 20 400-419 111 111 111 208 208 208 F F F 21 420-439 107 107 107 203 203 203 F F F 22 440-459 114 114 114 214 214 214 F F F 23 460-479 113 113 113 213 213 213 T T T 24 480-499 114 114 114 214 214 214 T T T 25 500-519 114 114 114 214 214 214 T T T 26 520-539 111 111 111 208 208 208 F F F 27 540-559 114 114 114 214 214 214 F F F 28 560-579 111 111 111 208 208 208 F F F 29 580-599 113 113 113 213 213 213 F F F Max gen: 114, Max flush: 214 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 215 215 215 F F F 1 020-039 115 115 115 215 215 215 F F F 2 040-059 110 110 110 207 207 207 F F F 3 060-079 108 108 108 205 205 205 F F F 4 080-099 108 108 108 205 205 205 F F F 5 100-119 113 113 113 213 213 213 F F F 6 120-139 113 113 113 213 213 213 F F F 7 140-159 111 111 111 208 208 208 F F F 8 160-179 112 112 112 211 211 211 F F F 9 180-199 108 108 108 204 204 204 F F F 10 200-219 114 114 114 214 214 214 F F F 11 220-239 105 105 105 198 198 198 F F F 12 240-259 98 98 98 190 190 190 F F F 13 260-279 110 110 110 207 207 207 F F F 14 280-299 115 115 115 215 215 215 F F F 15 300-319 111 111 111 208 208 208 F F F 16 320-339 89 89 89 173 173 173 F F F 17 340-359 95 95 95 186 186 186 F F F 18 360-379 113 113 113 213 213 213 F F F 19 380-399 113 113 113 213 213 213 F F F 20 400-419 111 111 111 208 208 208 F F F 21 420-439 107 107 107 203 203 203 F F F 22 440-459 114 114 114 214 214 214 F F F 23 460-479 115 115 115 215 215 215 F F F 24 480-499 115 115 115 215 215 215 F F F 25 500-519 115 115 115 215 215 215 F F F 26 520-539 111 111 111 208 208 208 F F F 27 540-559 114 114 114 214 214 214 F F F 28 560-579 111 111 111 208 208 208 F F F 29 580-599 113 113 113 213 213 213 F F F Max gen: 115, Max flush: 215 Begin loop 58 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.338313832Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.383192526Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.383755738Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.383791279Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.383806339Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.3838205Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651} {"msg":"Crucible 218a31fb-a272-464f-a107-6ebcb4e810b4 has session id: 9cbcb2ba-9e18-471e-b726-a46208ba5fe3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.384232479Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"Upstairs opts: Upstairs UUID: 218a31fb-a272-464f-a107-6ebcb4e810b4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.38427824Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"Crucible stats registered with UUID: 218a31fb-a272-464f-a107-6ebcb4e810b4","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.3843008Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.41472095Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.414753331Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651} {"msg":"218a31fb-a272-464f-a107-6ebcb4e810b4 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.414783291Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.414800472Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"client":"0","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.414830123Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"client":"0","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.414847903Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"client":"1","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.414873313Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"client":"1","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.414905294Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"client":"2","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.414937855Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"client":"2","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.414963585Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"":"io task","client":"2","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.414996546Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"":"io task","client":"0","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.415031767Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"":"io task","client":"1","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"ds_connection connected from Ok(127.0.0.1:52873)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.415066257Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"":"io task","client":"2","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"ds_connection connected from Ok(127.0.0.1:37471)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.415100098Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"":"io task","client":"0","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"ds_connection connected from Ok(127.0.0.1:43197)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.415135349Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"":"io task","client":"1","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.415874665Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"client":"0","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.415904786Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"client":"0","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.415920536Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"client":"1","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.415934816Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"client":"1","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.415948917Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"client":"2","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.415996108Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"client":"2","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.416483878Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"client":"0","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.416514369Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"client":"1","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.416634291Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"client":"2","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.417104332Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"client":"1","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.417132782Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"client":"1","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.417147553Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.417160083Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"client":"0","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.417173823Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"client":"0","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.417186983Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.417199144Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"client":"2","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.417216434Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"client":"2","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"[0]R flush_numbers[0..12]: [215, 215, 207, 205, 205, 213, 213, 208, 211, 204, 214, 198]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.417231175Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"":"downstairs","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"[0]R generation[0..12]: [115, 115, 110, 108, 108, 113, 113, 111, 112, 108, 114, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.417245625Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"":"downstairs","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.417259915Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"":"downstairs","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"[1]R flush_numbers[0..12]: [215, 215, 207, 205, 205, 213, 213, 208, 211, 204, 214, 198]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.417273996Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"":"downstairs","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"[1]R generation[0..12]: [115, 115, 110, 108, 108, 113, 113, 111, 112, 108, 114, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.417287806Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"":"downstairs","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.417301716Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"":"downstairs","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"[2]R flush_numbers[0..12]: [215, 215, 207, 205, 205, 213, 213, 208, 211, 204, 214, 198]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.417315566Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"":"downstairs","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"[2]R generation[0..12]: [115, 115, 110, 108, 108, 113, 113, 111, 112, 108, 114, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.417329427Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"":"downstairs","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.417343347Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"":"downstairs","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.417356407Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"":"downstairs","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.417369727Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"":"downstairs","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"Next flush: 216","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.417382828Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"":"downstairs","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:54:23.417395888Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"":"downstairs","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.417417259Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.417429459Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"client":"0","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.417442629Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"client":"1","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.417455829Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"client":"2","":"downstairs","session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.41746916Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.41748158Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"218a31fb-a272-464f-a107-6ebcb4e810b4 is now active with session: 9cbcb2ba-9e18-471e-b726-a46208ba5fe3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.41749437Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.41750655Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651,"session_id":"9cbcb2ba-9e18-471e-b726-a46208ba5fe3"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:23.417519261Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 131 len 4096 data: 8 02/30 Write block 84 len 36864 data: 3 4 4 5 6 9 8 11 11 03/30 Flush 04/30 Write block 265 len 32768 data: 9 11 11 11 13 16 14 14 05/30 Write block 529 len 40960 data: 13 11 8 7 6 7 6 5 8 11 06/30 Read block 24 len 16384 07/30 Read block 153 len 28672 08/30 Read block 543 len 8192 09/30 Write block 380 len 8192 data: 10 10 10/30 Read block 530 len 20480 11/30 Write block 547 len 12288 data: 13 13 11 12/30 Flush 13/30 Flush 14/30 Flush 15/30 Write block 354 len 28672 data: 5 7 7 7 9 9 7 16/30 Read block 224 len 20480 17/30 Write block 409 len 40960 data: 10 13 10 9 9 7 7 5 5 2 18/30 Read block 494 len 8192 19/30 Write block 197 len 4096 data: 14 20/30 Write block 176 len 4096 data: 10 21/30 Read block 350 len 36864 22/30 Write block 378 len 40960 data: 10 11 11 11 9 7 6 6 5 10 23/30 Read block 128 len 4096 24/30 Write block 214 len 8192 data: 12 12 25/30 Read block 490 len 32768 26/30 Read block 404 len 20480 27/30 Read block 448 len 36864 28/30 Write block 367 len 40960 data: 8 8 9 9 8 8 7 6 7 9 29/30 Write block 403 len 16384 data: 10 11 10 10 30/30 Read block 169 len 20480 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 7 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 4 Sent Done Done false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 14 16 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:54:23.436989113Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1651} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 215 215 215 F F F 1 020-039 115 115 115 215 215 215 F F F 2 040-059 110 110 110 207 207 207 F F F 3 060-079 108 108 108 205 205 205 F F F 4 080-099 116 116 116 216 216 216 F F F 5 100-119 113 113 113 213 213 213 F F F 6 120-139 116 116 116 216 216 216 F F F 7 140-159 111 111 111 208 208 208 F F F 8 160-179 112 112 112 211 211 211 T T T 9 180-199 108 108 108 204 204 204 T T T 10 200-219 114 114 114 214 214 214 T T T 11 220-239 105 105 105 198 198 198 F F F 12 240-259 98 98 98 190 190 190 F F F 13 260-279 116 116 116 217 217 217 F F F 14 280-299 115 115 115 215 215 215 F F F 15 300-319 111 111 111 208 208 208 F F F 16 320-339 89 89 89 173 173 173 F F F 17 340-359 95 95 95 186 186 186 T T T 18 360-379 113 113 113 213 213 213 T T T 19 380-399 116 116 116 217 217 217 T T T 20 400-419 111 111 111 208 208 208 T T T 21 420-439 107 107 107 203 203 203 F F F 22 440-459 114 114 114 214 214 214 F F F 23 460-479 115 115 115 215 215 215 F F F 24 480-499 115 115 115 215 215 215 F F F 25 500-519 115 115 115 215 215 215 F F F 26 520-539 116 116 116 217 217 217 F F F 27 540-559 116 116 116 217 217 217 F F F 28 560-579 111 111 111 208 208 208 F F F 29 580-599 113 113 113 213 213 213 F F F Max gen: 116, Max flush: 217 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 215 215 215 F F F 1 020-039 115 115 115 215 215 215 F F F 2 040-059 110 110 110 207 207 207 F F F 3 060-079 108 108 108 205 205 205 F F F 4 080-099 116 116 116 216 216 216 F F F 5 100-119 113 113 113 213 213 213 F F F 6 120-139 116 116 116 216 216 216 F F F 7 140-159 111 111 111 208 208 208 F F F 8 160-179 117 117 117 218 218 218 F F F 9 180-199 117 117 117 218 218 218 F F F 10 200-219 117 117 117 218 218 218 F F F 11 220-239 105 105 105 198 198 198 F F F 12 240-259 98 98 98 190 190 190 F F F 13 260-279 116 116 116 217 217 217 F F F 14 280-299 115 115 115 215 215 215 F F F 15 300-319 111 111 111 208 208 208 F F F 16 320-339 89 89 89 173 173 173 F F F 17 340-359 117 117 117 218 218 218 F F F 18 360-379 117 117 117 218 218 218 F F F 19 380-399 117 117 117 218 218 218 F F F 20 400-419 117 117 117 218 218 218 F F F 21 420-439 107 107 107 203 203 203 F F F 22 440-459 114 114 114 214 214 214 F F F 23 460-479 115 115 115 215 215 215 F F F 24 480-499 115 115 115 215 215 215 F F F 25 500-519 115 115 115 215 215 215 F F F 26 520-539 116 116 116 217 217 217 F F F 27 540-559 116 116 116 217 217 217 F F F 28 560-579 111 111 111 208 208 208 F F F 29 580-599 113 113 113 213 213 213 F F F Max gen: 117, Max flush: 218 Begin loop 59 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.717371318Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.761535346Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.762161449Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.76219671Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.76221141Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.762225921Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659} {"msg":"Crucible c9d938a8-fea8-4205-b4d1-a4fa75b3b244 has session id: 399ad080-86ad-4e37-be7d-262c5f3ee645","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.762687971Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"Upstairs opts: Upstairs UUID: c9d938a8-fea8-4205-b4d1-a4fa75b3b244, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.762734272Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"Crucible stats registered with UUID: c9d938a8-fea8-4205-b4d1-a4fa75b3b244","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.762759902Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.793584351Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.793623241Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659} {"msg":"c9d938a8-fea8-4205-b4d1-a4fa75b3b244 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.793649262Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.793666413Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"client":"0","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.793695953Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"client":"0","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.793713914Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"client":"1","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.793731304Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"client":"1","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.793747914Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"client":"2","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.793764655Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"client":"2","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.793781025Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"":"io task","client":"2","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.793804065Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"":"io task","client":"0","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.793836646Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"":"io task","client":"1","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"ds_connection connected from Ok(127.0.0.1:63445)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.794073281Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"":"io task","client":"2","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"ds_connection connected from Ok(127.0.0.1:34636)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.794117942Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"":"io task","client":"0","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"ds_connection connected from Ok(127.0.0.1:60963)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.794138293Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"":"io task","client":"1","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.794851948Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"client":"0","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.794880829Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"client":"0","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.794900749Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"client":"1","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.794915149Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"client":"1","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.795005451Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"client":"2","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.795038482Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"client":"2","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.795597074Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"client":"0","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.795653946Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"client":"2","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.795673806Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"client":"1","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.796229458Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"client":"1","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.796259979Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"client":"1","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.796275049Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.796287739Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"client":"2","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.79630165Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"client":"2","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.79631542Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.79632789Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"client":"0","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.796341551Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"client":"0","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"[0]R flush_numbers[0..12]: [215, 215, 207, 205, 216, 213, 216, 208, 218, 218, 218, 198]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.796359901Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"":"downstairs","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"[0]R generation[0..12]: [115, 115, 110, 108, 116, 113, 116, 111, 117, 117, 117, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.796374741Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"":"downstairs","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.796389152Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"":"downstairs","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"[1]R flush_numbers[0..12]: [215, 215, 207, 205, 216, 213, 216, 208, 218, 218, 218, 198]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.796403172Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"":"downstairs","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"[1]R generation[0..12]: [115, 115, 110, 108, 116, 113, 116, 111, 117, 117, 117, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.796427722Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"":"downstairs","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.796456683Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"":"downstairs","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"[2]R flush_numbers[0..12]: [215, 215, 207, 205, 216, 213, 216, 208, 218, 218, 218, 198]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.796478654Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"":"downstairs","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"[2]R generation[0..12]: [115, 115, 110, 108, 116, 113, 116, 111, 117, 117, 117, 105]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.796493834Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"":"downstairs","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":Wait for a query_work_queue command to finish before sending IO "crucible","level":30,"time":"2024-11-28T04:54:27.796528555Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"":"downstairs","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.796557225Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"":"downstairs","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.796577756Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"":"downstairs","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"Next flush: 219","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.796591636Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"":"downstairs","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.796609866Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"":"downstairs","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.796634327Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.796663647Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"client":"0","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.796683988Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"client":"1","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.796701318Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"client":"2","":"downstairs","session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.796726129Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.79675174Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"c9d938a8-fea8-4205-b4d1-a4fa75b3b244 is now active with session: 399ad080-86ad-4e37-be7d-262c5f3ee645","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.79677614Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.79679035Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659,"session_id":"399ad080-86ad-4e37-be7d-262c5f3ee645"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:27.796812601Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 103 len 16384 data: 10 9 8 7 02/30 Read block 109 len 4096 03/30 Flush 04/30 Write block 113 len 24576 data: 11 12 12 10 10 9 05/30 Read block 484 len 4096 06/30 Read block 351 len 40960 07/30 Read block 421 len 36864 08/30 Write block 48 len 24576 data: 9 9 12 11 11 12 09/30 Write block 470 len 32768 data: 8 8 6 6 10 11 13 12 10/30 Write block 218 len 36864 data: 10 9 7 10 10 11 8 7 5 11/30 Write block 419 len 24576 data: 4 5 7 8 7 7 12/30 Read block 493 len 36864 13/30 Read block 587 len 8192 14/30 Write block 137 len 40960 data: 5 6 6 6 8 9 9 9 9 10 15/30 Flush 16/30 Write block 95 len 20480 data: 12 9 6 8 7 17/30 Write block 50 len 16384 data: 13 12 12 13 18/30 Read block 510 len 28672 19/30 Flush 20/30 Write block 194 len 20480 data: 13 15 11 15 13 21/30 Write block 224 len 40960 data: 9 8 6 8 6 7 7 9 11 12 22/30 Read block 358 len 24576 23/30 Read block 95 len 24576 24/30 Write block 362 len 28672 data: 9 9 6 6 8 9 9 25/30 Flush 26/30 Write block 252 len 20480 data: 11 11 11 9 8 27/30 Read block 266 len 40960 28/30 Read block 65 len 20480 29/30 Read block 266 len 20480 30/30 Write block 335 len 32768 data: 11 11 10 12 10 10 9 9 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 5 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:54:27.817251464Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1659} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 215 215 215 F F F 1 020-039 115 115 115 215 215 215 F F F 2 040-059 118 118 118 221 221 221 F F F 3 060-079 108 108 108 205 205 205 F F F 4 080-099 118 118 118 221 221 221 F F F 5 100-119 118 118 118 220 220 220 F F F 6 120-139 118 118 118 220 220 220 F F F 7 140-159 118 118 118 220 220 220 F F F 8 160-179 117 117 117 218 218 218 F F F 9 180-199 118 118 118 222 222 222 F F F 10 200-219 118 118 118 220 220 220 F F F 11 220-239 118 118 118 222 222 222 F F F 12 240-259 98 98 98 190 190 190 T T T 13 260-279 116 116 116 217 217 217 F F F 14 280-299 115 115 115 215 215 215 F F F 15 300-319 111 111 111 208 208 208 F F F 16 320-339 89 89 89 173 173 173 T T T 17 340-359 117 117 117 218 218 218 T T T 18 360-379 118 118 118 222 222 222 F F F 19 380-399 117 117 117 218 218 218 F F F 20 400-419 118 118 118 220 220 220 F F F 21 420-439 118 118 118 220 220 220 F F F 22 440-459 114 114 114 214 214 214 F F F 23 460-479 118 118 118 220 220 220 F F F 24 480-499 115 115 115 215 215 215 F F F 25 500-519 115 115 115 215 215 215 F F F 26 520-539 116 116 116 217 217 217 F F F 27 540-559 116 116 116 217 217 217 F F F 28 560-579 111 111 111 208 208 208 F F F 29 580-599 113 113 113 213 213 213 F F F Max gen: 118, Max flush: 222 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 215 215 215 F F F 1 020-039 115 115 115 215 215 215 F F F 2 040-059 118 118 118 221 221 221 F F F 3 060-079 108 108 108 205 205 205 F F F 4 080-099 118 118 118 221 221 221 F F F 5 100-119 118 118 118 220 220 220 F F F 6 120-139 118 118 118 220 220 220 F F F 7 140-159 118 118 118 220 220 220 F F F 8 160-179 117 117 117 218 218 218 F F F 9 180-199 118 118 118 222 222 222 F F F 10 200-219 118 118 118 220 220 220 F F F 11 220-239 118 118 118 222 222 222 F F F 12 240-259 119 119 119 223 223 223 F F F 13 260-279 116 116 116 217 217 217 F F F 14 280-299 115 115 115 215 215 215 F F F 15 300-319 111 111 111 208 208 208 F F F 16 320-339 119 119 119 223 223 223 F F F 17 340-359 119 119 119 223 223 223 F F F 18 360-379 118 118 118 222 222 222 F F F 19 380-399 117 117 117 218 218 218 F F F 20 400-419 118 118 118 220 220 220 F F F 21 420-439 118 118 118 220 220 220 F F F 22 440-459 114 114 114 214 214 214 F F F 23 460-479 118 118 118 220 220 220 F F F 24 480-499 115 115 115 215 215 215 F F F 25 500-519 115 115 115 215 215 215 F F F 26 520-539 116 116 116 217 217 217 F F F 27 540-559 116 116 116 217 217 217 F F F 28 560-579 111 111 111 208 208 208 F F F 29 580-599 113 113 113 213 213 213 F F F Max gen: 119, Max flush: 223 Begin loop 60 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.073852943Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.118202325Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.118802888Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.118838838Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.118853389Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.118868109Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667} {"msg":"Crucible dd9fb3e8-8eb2-4b72-b92f-e322aa2bae11 has session id: 9b09b8c9-321e-48a8-9a73-6ba22aef777a","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.119315089Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"Upstairs opts: Upstairs UUID: dd9fb3e8-8eb2-4b72-b92f-e322aa2bae11, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.11936015Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"Crucible stats registered with UUID: dd9fb3e8-8eb2-4b72-b92f-e322aa2bae11","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.11938545Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.150684119Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.150718Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667} {"msg":"dd9fb3e8-8eb2-4b72-b92f-e322aa2bae11 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.150768251Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.150787752Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"client":"0","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.150816002Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"client":"0","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.150837782Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"client":"1","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.150855103Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"client":"1","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.150871633Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"client":"2","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.150888364Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"client":"2","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.150916484Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"":"io task","client":"2","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.150948415Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"":"io task","client":"0","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.150979076Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"":"io task","client":"1","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"ds_connection connected from Ok(127.0.0.1:64889)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.151012026Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"":"io task","client":"0","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"ds_connection connected from Ok(127.0.0.1:65008)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.151046437Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"":"io task","client":"2","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"ds_connection connected from Ok(127.0.0.1:57313)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.151081758Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"":"io task","client":"1","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.151954607Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"client":"2","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.151998718Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"client":"2","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.152031919Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"client":"0","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.152060729Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"client":"0","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.15207626Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"client":"1","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.1520907Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"client":"1","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.152636861Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"client":"0","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.152665212Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"client":"1","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.152680653Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"client":"2","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.153218674Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"client":"1","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.153258055Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"client":"1","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.153283766Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.153305916Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"client":"2","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.153329957Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"client":"2","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.153355197Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.153377738Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"client":"0","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.153400818Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"client":"0","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"[0]R flush_numbers[0..12]: [215, 215, 221, 205, 221, 220, 220, 220, 218, 222, 220, 222]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.153425339Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"":"downstairs","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"[0]R generation[0..12]: [115, 115, 118, 108, 118, 118, 118, 118, 117, 118, 118, 118]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.153451829Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"":"downstairs","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.15347939Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"":"downstairs","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"[1]R flush_numbers[0..12]: [215, 215, 221, 205, 221, 220, 220, 220, 218, 222, 220, 222]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.15350908Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"":"downstairs","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"[1]R generation[0..12]: [115, 115, 118, 108, 118, 118, 118, 118, 117, 118, 118, 118]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:54:32.153550061Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"":"downstairs","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.153582192Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"":"downstairs","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"[2]R flush_numbers[0..12]: [215, 215, 221, 205, 221, 220, 220, 220, 218, 222, 220, 222]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.153613843Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"":"downstairs","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"[2]R generation[0..12]: [115, 115, 118, 108, 118, 118, 118, 118, 117, 118, 118, 118]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.153643974Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"":"downstairs","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.153675164Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"":"downstairs","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.153699735Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"":"downstairs","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.153714275Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"":"downstairs","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"Next flush: 224","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.153727615Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"":"downstairs","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.153740756Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"":"downstairs","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.153754026Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.153766146Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"client":"0","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.153779466Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"client":"1","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.153792957Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"client":"2","":"downstairs","session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"All required reconciliation work is completed","v":0,"name":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 crucible","level":30,"time":"2024-11-28T04:54:32.153813827Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.153826447Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"dd9fb3e8-8eb2-4b72-b92f-e322aa2bae11 is now active with session: 9b09b8c9-321e-48a8-9a73-6ba22aef777a","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.153839378Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.153851658Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667,"session_id":"9b09b8c9-321e-48a8-9a73-6ba22aef777a"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:32.153864228Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 161 len 16384 data: 8 8 8 8 02/30 Read block 542 len 28672 03/30 Read block 440 len 40960 04/30 Read block 589 len 20480 05/30 Read block 191 len 8192 06/30 Write block 539 len 40960 data: 11 8 7 9 9 10 15 14 14 14 07/30 Read block 208 len 24576 08/30 Read block 450 len 16384 09/30 Flush 10/30 Write block 130 len 36864 data: 9 9 9 7 7 9 7 6 7 11/30 Read block 162 len 40960 12/30 Read block 100 len 36864 13/30 Read block 157 len 4096 14/30 Read block 224 len 20480 15/30 Read block 564 len 28672 16/30 Read block 300 len 8192 17/30 Read block 275 len 32768 18/30 Read block 587 len 12288 19/30 Read block 266 len 32768 20/30 Read block 469 len 8192 21/30 Read block 7 len 24576 22/30 Flush 23/30 Write block 317 len 28672 data: 4 4 5 4 5 5 7 24/30 Write block 441 len 32768 data: 11 9 10 8 8 6 6 7 25/30 Read block 476 len 12288 26/30 Write block 440 len 24576 data: 10 12 10 11 9 9 27/30 Write block 104 len 36864 data: 10 9 8 8 8 8 10 10 11 28/30 Read block 322 len 20480 29/30 Write block 466 len 12288 data: 6 7 8 30/30 Write block 400 len 8192 data: 10 11 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 7 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:54:32.17424861Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1667} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 215 215 215 F F F 1 020-039 115 115 115 215 215 215 F F F 2 040-059 118 118 118 221 221 221 F F F 3 060-079 108 108 108 205 205 205 F F F 4 080-099 118 118 118 221 221 221 F F F 5 100-119 118 118 118 220 220 220 T T T 6 120-139 120 120 120 225 225 225 F F F 7 140-159 118 118 118 220 220 220 F F F 8 160-179 120 120 120 224 224 224 F F F 9 180-199 118 118 118 222 222 222 F F F 10 200-219 118 118 118 220 220 220 F F F 11 220-239 118 118 118 222 222 222 F F F 12 240-259 119 119 119 223 223 223 F F F 13 260-279 116 116 116 217 217 217 F F F 14 280-299 115 115 115 215 215 215 F F F 15 300-319 111 111 111 208 208 208 T T T 16 320-339 119 119 119 223 223 223 T T T 17 340-359 119 119 119 223 223 223 F F F 18 360-379 118 118 118 222 222 222 F F F 19 380-399 117 117 117 218 218 218 F F F 20 400-419 118 118 118 220 220 220 T T T 21 420-439 118 118 118 220 220 220 F F F 22 440-459 114 114 114 214 214 214 T T T 23 460-479 118 118 118 220 220 220 T T T 24 480-499 115 115 115 215 215 215 F F F 25 500-519 115 115 115 215 215 215 F F F 26 520-539 120 120 120 224 224 224 F F F 27 540-559 120 120 120 224 224 224 F F F 28 560-579 111 111 111 208 208 208 F F F 29 580-599 113 113 113 213 213 213 F F F Max gen: 120, Max flush: 225 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 215 215 215 F F F 1 020-039 115 115 115 215 215 215 F F F 2 040-059 118 118 118 221 221 221 F F F 3 060-079 108 108 108 205 205 205 F F F 4 080-099 118 118 118 221 221 221 F F F 5 100-119 121 121 121 226 226 226 F F F 6 120-139 120 120 120 225 225 225 F F F 7 140-159 118 118 118 220 220 220 F F F 8 160-179 120 120 120 224 224 224 F F F 9 180-199 118 118 118 222 222 222 F F F 10 200-219 118 118 118 220 220 220 F F F 11 220-239 118 118 118 222 222 222 F F F 12 240-259 119 119 119 223 223 223 F F F 13 260-279 116 116 116 217 217 217 F F F 14 280-299 115 115 115 215 215 215 F F F 15 300-319 121 121 121 226 226 226 F F F 16 320-339 121 121 121 226 226 226 F F F 17 340-359 119 119 119 223 223 223 F F F 18 360-379 118 118 118 222 222 222 F F F 19 380-399 117 117 117 218 218 218 F F F 20 400-419 121 121 121 226 226 226 F F F 21 420-439 118 118 118 220 220 220 F F F 22 440-459 121 121 121 226 226 226 F F F 23 460-479 121 121 121 226 226 226 F F F 24 480-499 115 115 115 215 215 215 F F F 25 500-519 115 115 115 215 215 215 F F F 26 520-539 120 120 120 224 224 224 F F F 27 540-559 120 120 120 224 224 224 F F F 28 560-579 111 111 111 208 208 208 F F F 29 580-599 113 113 113 213 213 213 F F F Max gen: 121, Max flush: 226 Begin loop 61 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.446499579Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.490466433Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.491106576Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.491142127Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.491160557Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.491175458Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675} {"msg":"Crucible 56d69080-dc5e-4775-aef3-1cea7f8a7649 has session id: 222ec68b-efe2-4b08-8e5f-508061d62e40","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.491606317Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"Upstairs opts: Upstairs UUID: 56d69080-dc5e-4775-aef3-1cea7f8a7649, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.491654668Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"Crucible stats registered with UUID: 56d69080-dc5e-4775-aef3-1cea7f8a7649","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.491680629Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.522365594Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.522391805Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675} {"msg":"56d69080-dc5e-4775-aef3-1cea7f8a7649 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.522466447Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.522488217Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"client":"0","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.522516818Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"client":"0","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.522535568Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"client":"1","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.522552478Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"client":"1","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.522568969Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"client":"2","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.522590309Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"client":"2","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.52260744Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.52262605Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.522644731Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"ds_connection connected from Ok(127.0.0.1:41546)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.522697972Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"ds_connection connected from Ok(127.0.0.1:38370)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.522735562Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"ds_connection connected from Ok(127.0.0.1:57535)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.522904656Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.523394957Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"client":"2","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.523433798Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"client":"2","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.523464188Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"client":"0","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.523492249Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"client":"0","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.52351883Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"client":"1","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.52354594Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"client":"1","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.523978129Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"client":"2","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.524121883Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"client":"1","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.524234545Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"client":"0","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.524620203Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"client":"2","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.524646264Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"client":"2","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.524660574Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.524754946Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"client":"1","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.524787387Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"client":"1","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.524813858Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.524828208Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"client":"0","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.524841768Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"client":"0","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"[0]R flush_numbers[0..12]: [215, 215, 221, 205, 221, 226, 225, 220, 224, 222, 220, 222]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.524855608Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"":"downstairs","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"[0]R generation[0..12]: [115, 115, 118, 108, 118, 121, 120, 118, 120, 118, 118, 118]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.524869529Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"":"downstairs","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.524883659Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"":"downstairs","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"[1]R flush_numbers[0..12]: [215, 215, 221, 205, 221, 226, 225, 220, 224, 222, 220, 222]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.524897559Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"":"downstairs","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"[1]R generation[0..12]: [115, 115, 118, 108, 118, 121, 120, 118, 120, 118, 118, 118]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.52491143Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"":"downstairs","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.52492532Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"":"downstairs","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"[2]R flush_numbers[0..12]: [215, 215, 221, 205, 221, 226, 225, 220, 224, 222, 220, 222]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.5249429Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"":"downstairs","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"[2]R generation[0..12]: [115, 115, 118, 108, 118, 121, 120, 118, 120, 118, 118, 118]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.524957441Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"":"downstairs","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.524971681Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"":"downstairs","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:54:36.524984911Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"":"downstairs","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.525027552Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"":"downstairs","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"Next flush: 227","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.525050173Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"":"downstairs","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.525070423Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"":"downstairs","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.525095854Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.525118464Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"client":"0","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.525136374Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"client":"1","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.525149925Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"client":"2","":"downstairs","session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.525163235Z","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.525179615Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"56d69080-dc5e-4775-aef3-1cea7f8a7649 is now active with session: 222ec68b-efe2-4b08-8e5f-508061d62e40","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.525192576Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.525205436Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675,"session_id":"222ec68b-efe2-4b08-8e5f-508061d62e40"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:36.525220436Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 409 len 32768 data: 11 14 11 10 10 8 8 6 02/30 Read block 31 len 8192 03/30 Read block 474 len 20480 04/30 Read block 73 len 20480 05/30 Flush 06/30 Write block 497 len 20480 data: 13 15 14 17 14 07/30 Write block 222 len 36864 data: 11 12 10 9 7 9 7 8 8 08/30 Flush 09/30 Write block 66 len 16384 data: 10 12 12 12 10/30 Write block 238 len 12288 data: 7 6 10 11/30 Write block 108 len 24576 data: 9 9 11 11 12 12 12/30 Flush 13/30 Write block 233 len 8192 data: 13 13 14/30 Read block 38 len 8192 15/30 Write block 365 len 20480 data: 7 9 10 10 10 16/30 Read block 460 len 40960 17/30 Read block 76 len 16384 18/30 Write block 482 len 16384 data: 10 9 10 10 19/30 Write block 374 len 28672 data: 7 8 10 11 11 12 12 20/30 Read block 559 len 28672 21/30 Write block 199 len 20480 data: 11 12 11 13 11 22/30 Flush 23/30 Write block 282 len 12288 data: 8 7 10 24/30 Write block 82 len 4096 data: 7 25/30 Write block 571 len 40960 data: 12 12 14 15 15 10 9 9 8 6 26/30 Write block 1 len 12288 data: 4 5 5 27/30 Read block 502 len 8192 28/30 Read block 544 len 36864 29/30 Write block 555 len 16384 data: 12 11 9 9 30/30 Write block 387 len 20480 data: 11 10 10 8 8 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 3 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:54:36.54334434Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1675} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 215 215 215 T T T 1 020-039 115 115 115 215 215 215 F F F 2 040-059 118 118 118 221 221 221 F F F 3 060-079 122 122 122 229 229 229 F F F 4 080-099 118 118 118 221 221 221 T T T 5 100-119 122 122 122 229 229 229 F F F 6 120-139 120 120 120 225 225 225 F F F 7 140-159 118 118 118 220 220 220 F F F 8 160-179 120 120 120 224 224 224 F F F 9 180-199 122 122 122 230 230 230 F F F 10 200-219 122 122 122 230 230 230 F F F 11 220-239 122 122 122 230 230 230 F F F 12 240-259 122 122 122 229 229 229 F F F 13 260-279 116 116 116 217 217 217 F F F 14 280-299 115 115 115 215 215 215 T T T 15 300-319 121 121 121 226 226 226 F F F 16 320-339 121 121 121 226 226 226 F F F 17 340-359 119 119 119 223 223 223 F F F 18 360-379 122 122 122 230 230 230 F F F 19 380-399 122 122 122 230 230 230 T T T 20 400-419 122 122 122 227 227 227 F F F 21 420-439 118 118 118 220 220 220 F F F 22 440-459 121 121 121 226 226 226 F F F 23 460-479 121 121 121 226 226 226 F F F 24 480-499 122 122 122 230 230 230 F F F 25 500-519 122 122 122 228 228 228 F F F 26 520-539 120 120 120 224 224 224 F F F 27 540-559 120 120 120 224 224 224 T T T 28 560-579 111 111 111 208 208 208 T T T 29 580-599 113 113 113 213 213 213 T T T Max gen: 122, Max flush: 230 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 123 123 123 231 231 231 F F F 1 020-039 115 115 115 215 215 215 F F F 2 040-059 118 118 118 221 221 221 F F F 3 060-079 122 122 122 229 229 229 F F F 4 080-099 123 123 123 231 231 231 F F F 5 100-119 122 122 122 229 229 229 F F F 6 120-139 120 120 120 225 225 225 F F F 7 140-159 118 118 118 220 220 220 F F F 8 160-179 120 120 120 224 224 224 F F F 9 180-199 122 122 122 230 230 230 F F F 10 200-219 122 122 122 230 230 230 F F F 11 220-239 122 122 122 230 230 230 F F F 12 240-259 122 122 122 229 229 229 F F F 13 260-279 116 116 116 217 217 217 F F F 14 280-299 123 123 123 231 231 231 F F F 15 300-319 121 121 121 226 226 226 F F F 16 320-339 121 121 121 226 226 226 F F F 17 340-359 119 119 119 223 223 223 F F F 18 360-379 122 122 122 230 230 230 F F F 19 380-399 123 123 123 231 231 231 F F F 20 400-419 122 122 122 227 227 227 F F F 21 420-439 118 118 118 220 220 220 F F F 22 440-459 121 121 121 226 226 226 F F F 23 460-479 121 121 121 226 226 226 F F F 24 480-499 122 122 122 230 230 230 F F F 25 500-519 122 122 122 228 228 228 F F F 26 520-539 120 120 120 224 224 224 F F F 27 540-559 123 123 123 231 231 231 F F F 28 560-579 123 123 123 231 231 231 F F F 29 580-599 123 123 123 231 231 231 F F F Max gen: 123, Max flush: 231 Begin loop 62 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.817332445Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.860957181Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.861603635Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.861642516Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.861657316Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.861672866Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683} {"msg":"Crucible 69af1a1e-b96b-4971-9624-666a3a653668 has session id: 23e79dc7-dc20-4760-85b8-fefa1b70eaa3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.862199498Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"Upstairs opts: Upstairs UUID: 69af1a1e-b96b-4971-9624-666a3a653668, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.862234399Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"Crucible stats registered with UUID: 69af1a1e-b96b-4971-9624-666a3a653668","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.862250339Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.893917736Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.893962257Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683} {"msg":"69af1a1e-b96b-4971-9624-666a3a653668 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.894029399Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.894057949Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"client":"0","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.89408385Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"client":"0","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.89409882Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"client":"1","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.89411279Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"client":"1","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.894126121Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"client":"2","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.894139731Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"client":"2","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.894153111Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.894166942Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.894180682Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"ds_connection connected from Ok(127.0.0.1:44826)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.894274244Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"ds_connection connected from Ok(127.0.0.1:48098)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.894524719Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"ds_connection connected from Ok(127.0.0.1:44114)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.89455465Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.895127042Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"client":"0","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.895158453Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"client":"0","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.895176133Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"client":"2","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.895193394Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"client":"2","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.895270495Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"client":"1","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.895301586Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"client":"1","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.895781666Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"client":"2","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.89594356Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"client":"1","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.895983191Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"client":"0","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.896429251Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"client":"2","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.896458171Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"client":"2","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.896476541Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.896504322Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"client":"0","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.896518682Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"client":"0","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.896532183Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.896597834Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"client":"1","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.896617395Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"client":"1","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"[0]R flush_numbers[0..12]: [231, 215, 221, 229, 231, 229, 225, 220, 224, 230, 230, 230]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.896631535Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"":"downstairs","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"[0]R generation[0..12]: [123, 115, 118, 122, 123, 122, 120, 118, 120, 122, 122, 122]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.896645505Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"":"downstairs","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.896659685Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"":"downstairs","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"[1]R flush_numbers[0..12]: [231, 215, 221, 229, 231, 229, 225, 220, 224, 230, 230, 230]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.896673436Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"":"downstairs","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"[1]R generation[0..12]: [123, 115, 118, 122, 123, 122, 120, 118, 120, 122, 122, 122]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.896687306Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"":"downstairs","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.896701357Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"":"downstairs","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"[2]R flush_numbers[0..12]: [231, 215, 221, 229, 231, 229, 225, 220, 224, 230, 230, 230]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.896715247Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"":"downstairs","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"[2]R generation[0..12]: [123, 115, 118, 122, 123, 122, 120, 118, 120, 122, 122, 122]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.896729027Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"":"downstairs","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.896742977Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"":"downstairs","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.896759608Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"":"downstairs","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.896773538Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"":"downstairs","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"Next flush: 232","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.896786728Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"":"downstairs","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.896799988Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"":"downstairs","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.896813189Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":Wait for a query_work_queue command to finish before sending IO 1683,"session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.896833799Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"client":"0","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.89684701Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"client":"1","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.89686023Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"client":"2","":"downstairs","session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.8968736Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.896885941Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"69af1a1e-b96b-4971-9624-666a3a653668 is now active with session: 23e79dc7-dc20-4760-85b8-fefa1b70eaa3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.896898771Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.896910841Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683,"session_id":"23e79dc7-dc20-4760-85b8-fefa1b70eaa3"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:40.896923141Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 123 len 16384 data: 9 7 6 6 02/30 Write block 189 len 32768 data: 13 12 13 12 11 14 16 12 03/30 Write block 448 len 28672 data: 8 7 6 9 9 10 13 04/30 Read block 157 len 8192 05/30 Write block 393 len 40960 data: 9 7 8 8 7 7 10 11 12 10 06/30 Write block 67 len 24576 data: 13 13 13 11 15 14 07/30 Write block 375 len 40960 data: 9 11 12 12 13 13 12 10 8 7 08/30 Write block 445 len 16384 data: 10 7 7 9 09/30 Read block 479 len 28672 10/30 Read block 86 len 12288 11/30 Write block 92 len 16384 data: 12 11 10 13 12/30 Write block 147 len 36864 data: 12 12 14 14 13 13 12 12 13 13/30 Flush 14/30 Write block 19 len 32768 data: 13 11 13 11 12 11 10 10 15/30 Write block 109 len 24576 data: 10 12 12 13 13 13 16/30 Flush 17/30 Write block 330 len 36864 data: 8 9 8 8 11 12 12 11 13 18/30 Read block 405 len 40960 19/30 Read block 491 len 16384 20/30 Read block 162 len 24576 21/30 Read block 128 len 32768 22/30 Read block 222 len 20480 23/30 Read block 328 len 28672 24/30 Write block 27 len 8192 data: 10 10 25/30 Read block 263 len 28672 26/30 Flush 27/30 Write block 98 len 28672 data: 9 8 8 9 10 11 11 28/30 Write block 466 len 28672 data: 7 8 9 9 9 9 7 29/30 Read block 364 len 32768 30/30 Read block 563 len 32768 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 7 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 8 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 4 4 3 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:54:40.916865734Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1683} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 233 233 233 F F F 1 020-039 124 124 124 234 234 234 F F F 2 040-059 118 118 118 221 221 221 F F F 3 060-079 124 124 124 232 232 232 F F F 4 080-099 124 124 124 232 232 232 T T T 5 100-119 124 124 124 233 233 233 T T T 6 120-139 124 124 124 232 232 232 F F F 7 140-159 124 124 124 232 232 232 F F F 8 160-179 120 120 120 224 224 224 F F F 9 180-199 124 124 124 232 232 232 F F F 10 200-219 122 122 122 230 230 230 F F F 11 220-239 122 122 122 230 230 230 F F F 12 240-259 122 122 122 229 229 229 F F F 13 260-279 116 116 116 217 217 217 F F F 14 280-299 123 123 123 231 231 231 F F F 15 300-319 121 121 121 226 226 226 F F F 16 320-339 124 124 124 234 234 234 F F F 17 340-359 119 119 119 223 223 223 F F F 18 360-379 124 124 124 232 232 232 F F F 19 380-399 124 124 124 232 232 232 F F F 20 400-419 124 124 124 232 232 232 F F F 21 420-439 118 118 118 220 220 220 F F F 22 440-459 124 124 124 232 232 232 F F F 23 460-479 121 121 121 226 226 226 T T T 24 480-499 122 122 122 230 230 230 F F F 25 500-519 122 122 122 228 228 228 F F F 26 520-539 120 120 120 224 224 224 F F F 27 540-559 123 123 123 231 231 231 F F F 28 560-579 123 123 123 231 231 231 F F F 29 580-599 123 123 123 231 231 231 F F F Max gen: 124, Max flush: 234 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 233 233 233 F F F 1 020-039 124 124 124 234 234 234 F F F 2 040-059 118 118 118 221 221 221 F F F 3 060-079 124 124 124 232 232 232 F F F 4 080-099 125 125 125 235 235 235 F F F 5 100-119 125 125 125 235 235 235 F F F 6 120-139 124 124 124 232 232 232 F F F 7 140-159 124 124 124 232 232 232 F F F 8 160-179 120 120 120 224 224 224 F F F 9 180-199 124 124 124 232 232 232 F F F 10 200-219 122 122 122 230 230 230 F F F 11 220-239 122 122 122 230 230 230 F F F 12 240-259 122 122 122 229 229 229 F F F 13 260-279 116 116 116 217 217 217 F F F 14 280-299 123 123 123 231 231 231 F F F 15 300-319 121 121 121 226 226 226 F F F 16 320-339 124 124 124 234 234 234 F F F 17 340-359 119 119 119 223 223 223 F F F 18 360-379 124 124 124 232 232 232 F F F 19 380-399 124 124 124 232 232 232 F F F 20 400-419 124 124 124 232 232 232 F F F 21 420-439 118 118 118 220 220 220 F F F 22 440-459 124 124 124 232 232 232 F F F 23 460-479 125 125 125 235 235 235 F F F 24 480-499 122 122 122 230 230 230 F F F 25 500-519 122 122 122 228 228 228 F F F 26 520-539 120 120 120 224 224 224 F F F 27 540-559 123 123 123 231 231 231 F F F 28 560-579 123 123 123 231 231 231 F F F 29 580-599 123 123 123 231 231 231 F F F Max gen: 125, Max flush: 235 Begin loop 63 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.169323662Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.213655363Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.214246606Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.214281507Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.214295887Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.214310527Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691} {"msg":"Crucible fdb3808a-26a2-421e-bf0e-e9e86c231696 has session id: cfb872e9-19be-4a2b-9959-f15bd926517b","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.214805378Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"Upstairs opts: Upstairs UUID: fdb3808a-26a2-421e-bf0e-e9e86c231696, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.214839719Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"Crucible stats registered with UUID: fdb3808a-26a2-421e-bf0e-e9e86c231696","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.214857819Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.246037356Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.246071867Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691} {"msg":"fdb3808a-26a2-421e-bf0e-e9e86c231696 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.246140048Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.246162938Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"client":"0","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.246191539Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"client":"0","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.246208999Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"client":"1","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.2462259Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"client":"1","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.24624281Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"client":"2","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.2462598Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"client":"2","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.246276481Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.246293681Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.246323662Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"ds_connection connected from Ok(127.0.0.1:51006)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.246355013Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"ds_connection connected from Ok(127.0.0.1:51665)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.246385933Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"ds_connection connected from Ok(127.0.0.1:54735)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.246494046Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.247192891Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"client":"0","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.247221501Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"client":"0","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.247238302Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"client":"1","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.247264492Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"client":"1","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.247291353Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"client":"2","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.247317763Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"client":"2","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.247877276Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"client":"1","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.247920196Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"client":"2","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.247952697Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"client":"0","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.248502799Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"client":"1","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.24853964Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"client":"1","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.24855708Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.248571151Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"client":"0","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.248584861Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"client":"0","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.248598081Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.248614222Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"client":"2","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.248628202Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"client":"2","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"[0]R flush_numbers[0..12]: [233, 234, 221, 232, 235, 235, 232, 232, 224, 232, 230, 230]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.248642372Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"":"downstairs","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"[0]R generation[0..12]: [124, 124, 118, 124, 125, 125, 124, 124, 120, 124, 122, 122]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.248656372Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"":"downstairs","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.248670533Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"":"downstairs","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"[1]R flush_numbers[0..12]: [233, 234, 221, 232, 235, 235, 232, 232, 224, 232, 230, 230]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.248684313Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"":"downstairs","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"[1]R generation[0..12]: [124, 124, 118, 124, 125, 125, 124, 124, 120, 124, 122, 122]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.248697873Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"":"downstairs","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.248711864Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"":"downstairs","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"[2]R flush_numbers[0..12]: [233, 234, 221, 232, 235, 235, 232, 232, 224, 232, 230, 230]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.248725764Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"":"downstairs","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"[2]R generation[0..12]: [124, 124, 118, 124, 125, 125, 124, 124, 120, 124, 122, 122]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.248739524Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"":"downstairs","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:54:45.248753395Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"":"downstairs","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.248773625Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"":"downstairs","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.248798966Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"":"downstairs","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"Next flush: 236","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.248818606Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"":"downstairs","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.248835356Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"":"downstairs","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.248849227Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.248865617Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"client":"0","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.248890328Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"client":"1","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.248905958Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"client":"2","":"downstairs","session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.248919258Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.248940029Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"fdb3808a-26a2-421e-bf0e-e9e86c231696 is now active with session: cfb872e9-19be-4a2b-9959-f15bd926517b","v":0,"name":"crucible","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 level":30,"time":"2024-11-28T04:54:45.248966059Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.24897985Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691,"session_id":"cfb872e9-19be-4a2b-9959-f15bd926517b"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:45.24899281Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 435 len 40960 data: 10 10 6 7 8 11 13 11 12 10 02/30 Write block 214 len 36864 data: 13 13 12 9 11 10 8 11 12 03/30 Flush 04/30 Write block 26 len 40960 data: 11 11 11 9 6 7 7 6 7 9 05/30 Write block 479 len 24576 data: 10 11 10 11 10 11 06/30 Write block 472 len 16384 data: 8 7 11 12 07/30 Write block 552 len 12288 data: 13 12 11 08/30 Read block 381 len 12288 09/30 Read block 261 len 32768 10/30 Flush 11/30 Write block 124 len 36864 data: 8 7 7 6 11 8 10 10 10 12/30 Flush 13/30 Write block 244 len 20480 data: 9 11 13 13 14 14/30 Read block 492 len 40960 15/30 Read block 517 len 8192 16/30 Read block 162 len 36864 17/30 Write block 453 len 8192 data: 11 14 18/30 Read block 214 len 16384 19/30 Write block 374 len 24576 data: 8 10 12 13 13 14 20/30 Read block 590 len 4096 21/30 Flush 22/30 Write block 24 len 20480 data: 12 11 12 12 12 23/30 Read block 156 len 24576 24/30 Write block 66 len 40960 data: 11 14 14 14 12 16 15 15 17 16 25/30 Write block 420 len 40960 data: 6 8 9 8 8 7 7 6 8 10 26/30 Flush 27/30 Write block 10 len 40960 data: 8 10 7 8 7 9 9 8 10 14 28/30 Read block 12 len 20480 29/30 Write block 583 len 36864 data: 9 9 8 8 6 7 8 8 9 30/30 Read block 593 len 20480 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 10 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 9 Done Done Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 4 3 2 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:54:45.269576296Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1691} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 233 233 233 T T T 1 020-039 126 126 126 240 240 240 F F F 2 040-059 118 118 118 221 221 221 F F F 3 060-079 126 126 126 240 240 240 F F F 4 080-099 125 125 125 235 235 235 F F F 5 100-119 125 125 125 235 235 235 F F F 6 120-139 126 126 126 238 238 238 F F F 7 140-159 124 124 124 232 232 232 F F F 8 160-179 120 120 120 224 224 224 F F F 9 180-199 124 124 124 232 232 232 F F F 10 200-219 126 126 126 236 236 236 F F F 11 220-239 126 126 126 236 236 236 F F F 12 240-259 126 126 126 239 239 239 F F F 13 260-279 116 116 116 217 217 217 F F F 14 280-299 123 123 123 231 231 231 F F F 15 300-319 121 121 121 226 226 226 F F F 16 320-339 124 124 124 234 234 234 F F F 17 340-359 119 119 119 223 223 223 F F F 18 360-379 126 126 126 239 239 239 F F F 19 380-399 124 124 124 232 232 232 F F F 20 400-419 124 124 124 232 232 232 F F F 21 420-439 126 126 126 240 240 240 F F F 22 440-459 126 126 126 239 239 239 F F F 23 460-479 126 126 126 237 237 237 F F F 24 480-499 126 126 126 237 237 237 F F F 25 500-519 122 122 122 228 228 228 F F F 26 520-539 120 120 120 224 224 224 F F F 27 540-559 126 126 126 237 237 237 F F F 28 560-579 123 123 123 231 231 231 F F F 29 580-599 123 123 123 231 231 231 T T T Max gen: 126, Max flush: 240 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 127 127 127 241 241 241 F F F 1 020-039 126 126 126 240 240 240 F F F 2 040-059 118 118 118 221 221 221 F F F 3 060-079 126 126 126 240 240 240 F F F 4 080-099 125 125 125 235 235 235 F F F 5 100-119 125 125 125 235 235 235 F F F 6 120-139 126 126 126 238 238 238 F F F 7 140-159 124 124 124 232 232 232 F F F 8 160-179 120 120 120 224 224 224 F F F 9 180-199 124 124 124 232 232 232 F F F 10 200-219 126 126 126 236 236 236 F F F 11 220-239 126 126 126 236 236 236 F F F 12 240-259 126 126 126 239 239 239 F F F 13 260-279 116 116 116 217 217 217 F F F 14 280-299 123 123 123 231 231 231 F F F 15 300-319 121 121 121 226 226 226 F F F 16 320-339 124 124 124 234 234 234 F F F 17 340-359 119 119 119 223 223 223 F F F 18 360-379 126 126 126 239 239 239 F F F 19 380-399 124 124 124 232 232 232 F F F 20 400-419 124 124 124 232 232 232 F F F 21 420-439 126 126 126 240 240 240 F F F 22 440-459 126 126 126 239 239 239 F F F 23 460-479 126 126 126 237 237 237 F F F 24 480-499 126 126 126 237 237 237 F F F 25 500-519 122 122 122 228 228 228 F F F 26 520-539 120 120 120 224 224 224 F F F 27 540-559 126 126 126 237 237 237 F F F 28 560-579 123 123 123 231 231 231 F F F 29 580-599 127 127 127 241 241 241 F F F Max gen: 127, Max flush: 241 Begin loop 64 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.519340097Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:54:49.56143819Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.561813178Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.593304121Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.593883514Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.593919355Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.593934515Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.593948805Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699} {"msg":"Crucible 8b80a211-d594-42a2-a0ea-d3140996fb4e has session id: 34da2f10-4959-47c4-a83f-b3134b88358a","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.594415565Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"Upstairs opts: Upstairs UUID: 8b80a211-d594-42a2-a0ea-d3140996fb4e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.594458696Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"Crucible stats registered with UUID: 8b80a211-d594-42a2-a0ea-d3140996fb4e","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.594475647Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.624890117Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.624918957Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699} {"msg":"8b80a211-d594-42a2-a0ea-d3140996fb4e active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.624991099Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.625021469Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"client":"0","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.62504537Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"client":"0","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.62505998Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"client":"1","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.62507353Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"client":"1","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.625087071Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"client":"2","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.625100761Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"client":"2","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.625114081Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.625139312Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.625169753Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"ds_connection connected from Ok(127.0.0.1:46946)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.625260354Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"ds_connection connected from Ok(127.0.0.1:64544)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.625385007Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"ds_connection connected from Ok(127.0.0.1:45188)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.625423398Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.626049102Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"client":"2","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.626080332Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"client":"2","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.626105123Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"client":"1","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.626132813Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"client":"1","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.626160344Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"client":"0","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.626190085Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"client":"0","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.626652205Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"client":"2","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.626737817Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"client":"1","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.626812888Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"client":"0","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.627244017Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"client":"2","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.627269518Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"client":"2","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.627284258Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.627383521Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"client":"1","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.627417181Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"client":"1","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.627441092Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.627454482Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"client":"0","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.627468242Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"client":"0","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"[0]R flush_numbers[0..12]: [241, 240, 221, 240, 235, 235, 238, 232, 224, 232, 236, 236]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.627485673Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"":"downstairs","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"[0]R generation[0..12]: [127, 126, 118, 126, 125, 125, 126, 124, 120, 124, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.627500243Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"":"downstairs","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.627514594Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"":"downstairs","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"[1]R flush_numbers[0..12]: [241, 240, 221, 240, 235, 235, 238, 232, 224, 232, 236, 236]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.627528624Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"":"downstairs","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"[1]R generation[0..12]: [127, 126, 118, 126, 125, 125, 126, 124, 120, 124, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.627542614Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"":"downstairs","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.627556764Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"":"downstairs","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"[2]R flush_numbers[0..12]: [241, 240, 221, 240, 235, 235, 238, 232, 224, 232, 236, 236]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.627570735Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"":"downstairs","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"[2]R generation[0..12]: [127, 126, 118, 126, 125, 125, 126, 124, 120, 124, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.627584475Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"":"downstairs","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.627607336Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"":"downstairs","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.627641086Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"":"downstairs","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.627664327Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"":"downstairs","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"Next flush: 242","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.627678447Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"":"downstairs","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.627691627Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"":"downstairs","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.627704798Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.627719538Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"client":"0","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.627733188Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"client":"1","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.627746628Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"client":"2","":"downstairs","session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.627760059Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.627772489Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"8b80a211-d594-42a2-a0ea-d3140996fb4e is now active with session: 34da2f10-4959-47c4-a83f-b3134b88358a","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.627785539Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.62779777Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699,"session_id":"34da2f10-4959-47c4-a83f-b3134b88358a"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:49.62781037Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 463 len 32768 data: 11 8 6 8 9 10 10 10 02/30 Read block 165 len 24576 03/30 Write block 433 len 12288 data: 11 10 11 04/30 Read block 287 len 32768 05/30 Read block 162 len 12288 06/30 Flush 07/30 Write block 5 len 28672 data: 6 7 8 11 10 9 11 08/30 Read block 185 len 24576 09/30 Write block 195 len 20480 data: 17 13 16 14 12 10/30 Read block 533 len 20480 11/30 Write block 243 len 28672 data: 10 10 12 14 14 15 13 12/30 Read block 375 len 32768 13/30 Write block 425 len 24576 data: 8 8 7 9 11 10 14/30 Write block 390 len 28672 data: 9 9 9 10 8 9 9 15/30 Read block 513 len 8192 16/30 Read block 331 len 40960 17/30 Flush 18/30 Write block 229 len 40960 data: 9 9 10 12 14 14 11 12 9 8 19/30 Write block 15 len 40960 data: 10 10 9 11 15 12 14 12 13 13 20/30 Read block 516 len 40960 21/30 Read block 258 len 8192 22/30 Read block 389 len 28672 23/30 Read block 481 len 36864 24/30 Write block 133 len 24576 data: 8 8 10 8 7 8 25/30 Write block 311 len 24576 data: 4 4 5 9 8 5 26/30 Read block 440 len 24576 27/30 Read block 584 len 36864 28/30 Write block 564 len 8192 data: 3 3 29/30 Read block 313 len 16384 30/30 Read block 474 len 4096 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 10 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:54:49.64857549Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1699} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 243 243 243 T T T 1 020-039 126 126 126 240 240 240 T T T 2 040-059 118 118 118 221 221 221 F F F 3 060-079 126 126 126 240 240 240 F F F 4 080-099 125 125 125 235 235 235 F F F 5 100-119 125 125 125 235 235 235 F F F 6 120-139 126 126 126 238 238 238 T T T 7 140-159 124 124 124 232 232 232 F F F 8 160-179 120 120 120 224 224 224 F F F 9 180-199 128 128 128 243 243 243 F F F 10 200-219 126 126 126 236 236 236 F F F 11 220-239 126 126 126 236 236 236 T T T 12 240-259 128 128 128 243 243 243 F F F 13 260-279 116 116 116 217 217 217 F F F 14 280-299 123 123 123 231 231 231 F F F 15 300-319 121 121 121 226 226 226 T T T 16 320-339 124 124 124 234 234 234 F F F 17 340-359 119 119 119 223 223 223 F F F 18 360-379 126 126 126 239 239 239 F F F 19 380-399 128 128 128 243 243 243 F F F 20 400-419 124 124 124 232 232 232 F F F 21 420-439 128 128 128 243 243 243 F F F 22 440-459 126 126 126 239 239 239 F F F 23 460-479 128 128 128 242 242 242 F F F 24 480-499 126 126 126 237 237 237 F F F 25 500-519 122 122 122 228 228 228 F F F 26 520-539 120 120 120 224 224 224 F F F 27 540-559 126 126 126 237 237 237 F F F 28 560-579 123 123 123 231 231 231 T T T 29 580-599 127 127 127 241 241 241 F F F Max gen: 128, Max flush: 243 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 129 129 129 244 244 244 F F F 1 020-039 129 129 129 244 244 244 F F F 2 040-059 118 118 118 221 221 221 F F F 3 060-079 126 126 126 240 240 240 F F F 4 080-099 125 125 125 235 235 235 F F F 5 100-119 125 125 125 235 235 235 F F F 6 120-139 129 129 129 244 244 244 F F F 7 140-159 124 124 124 232 232 232 F F F 8 160-179 120 120 120 224 224 224 F F F 9 180-199 128 128 128 243 243 243 F F F 10 200-219 126 126 126 236 236 236 F F F 11 220-239 129 129 129 244 244 244 F F F 12 240-259 128 128 128 243 243 243 F F F 13 260-279 116 116 116 217 217 217 F F F 14 280-299 123 123 123 231 231 231 F F F 15 300-319 129 129 129 244 244 244 F F F 16 320-339 124 124 124 234 234 234 F F F 17 340-359 119 119 119 223 223 223 F F F 18 360-379 126 126 126 239 239 239 F F F 19 380-399 128 128 128 243 243 243 F F F 20 400-419 124 124 124 232 232 232 F F F 21 420-439 128 128 128 243 243 243 F F F 22 440-459 126 126 126 239 239 239 F F F 23 460-479 128 128 128 242 242 242 F F F 24 480-499 126 126 126 237 237 237 F F F 25 500-519 122 122 122 228 228 228 F F F 26 520-539 120 120 120 224 224 224 F F F 27 540-559 126 126 126 237 237 237 F F F 28 560-579 129 129 129 244 244 244 F F F 29 580-599 127 127 127 241 241 241 F F F Max gen: 129, Max flush: 244 Begin loop 65 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:53.951936213Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:53.996817277Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:53.997490531Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:53.997537642Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:53.997553493Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:53.997568543Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707} {"msg":"Crucible 874d68f2-f2c2-4006-a245-5f12d4bbfb16 has session id: 98314e63-d9f9-49ac-949e-30790dad0735","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:53.997976112Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"Upstairs opts: Upstairs UUID: 874d68f2-f2c2-4006-a245-5f12d4bbfb16, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:53.998014592Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"Crucible stats registered with UUID: 874d68f2-f2c2-4006-a245-5f12d4bbfb16","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:53.998030753Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.029882764Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.029926395Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707} {"msg":"874d68f2-f2c2-4006-a245-5f12d4bbfb16 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.029949645Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.029970036Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"client":"0","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.030000166Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"client":"0","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.030018067Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"client":"1","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.030035207Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"client":"1","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.030051647Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"client":"2","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.030068488Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"client":"2","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.030084998Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.030102228Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.030119399Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"ds_connection connected from Ok(127.0.0.1:46553)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.030250682Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"ds_connection connected from Ok(127.0.0.1:59869)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.030291173Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"ds_connection connected from Ok(127.0.0.1:43607)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.030321653Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.031029739Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"client":"0","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.031060509Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"client":"0","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.03107753Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"client":"2","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.03109209Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"client":"2","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.03110956Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"client":"1","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.031179552Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"client":"1","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.031618481Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"client":"0","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.031701773Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"client":"1","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.031738684Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"client":"2","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.032137553Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"client":"0","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.032163833Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"client":"0","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.032184984Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.032242975Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"client":"1","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.032272536Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"client":"1","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.032288366Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.032301216Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"client":"2","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.032315116Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"client":"2","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"[0]R flush_numbers[0..12]: [244, 244, 221, 240, 235, 235, 244, 232, 224, 243, 236, 244]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.032329287Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"":"downstairs","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"[0]R generation[0..12]: [129, 129, 118, 126, 125, 125, 129, 124, 120, 128, 126, 129]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.032343297Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"":"downstairs","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.032361477Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"":"downstairs","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"[1]R flush_numbers[0..12]: [244, 244, 221, 240, 235, 235, 244, 232, 224, 243, 236, 244]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.032375958Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"":"downstairs","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"[1]R generation[0..12]: [129, 129, 118, 126, 125, 125, 129, 124, 120, 128, 126, 129]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.032389948Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"":"downstairs","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.032404098Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"":"downstairs","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"[2]R flush_numbers[0..12]: [244, 244, 221, 240, 235, 235, 244, 232, 224, 243, 236, 244]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.032418299Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"":"downstairs","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"[2]R generation[0..12]: [129, 129, 118, 126, 125, 125, 129, 124, 120, 128, 126, 129]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.032432159Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"":"downstairs","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.032446109Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"":"downstairs","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.0324594Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"":"downstairs","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.03247294Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"":"downstairs","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"Next flush: 245","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.03248617Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"":"downstairs","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.032499371Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.032521651Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.032533851Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"client":"0","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.032547201Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"client":"1","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.032563482Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"client":"2","":"downstairs","session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.032577142Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.032589613Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"874d68f2-f2c2-4006-a245-5f12d4bbfb16 is now active with session: 98314e63-d9f9-49ac-949e-30790dad0735","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.032602623Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.032614993Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707,"session_id":"98314e63-d9f9-49ac-949e-30790dad0735"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:54.032627533Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 147 len 24576 data: 13 13 15 15 14 14 02/30 Read block 180 len 8192 03/30 Flush 04/30 Write block 243 len 36864 data: 11 11 13 15 15 16 14 12 11 05/30 Read block 57 len 24576 06/30 Write block 524 len 36864 data: 14 14 15 14 13 14 12 9 8 07/30 Write block 505 len 4096 data: 10 08/30 Read block 276 len 28672 09/30 Write block 115 len 28672 data: 13 11 11 10 8 6 8 10/30 Read block 352 len 16384 11/30 Write block 61 len 28672 data: 12 9 8 8 10 12 15 12/30 Read block 565 len 24576 13/30 Write block 210 len 20480 data: 11 12 12 11 14 14/30 Write block 279 len 8192 data: 7 6 15/30 Read block 303 len 32768 16/30 Write block 20 len 40960 data: 13 15 13 14 14 12 13 13 13 10 17/30 Write block 131 len 16384 data: 11 11 9 9 18/30 Read block 201 len 8192 19/30 Read block 83 len 32768 20/30 Read block 378 len 4096 21/30 Read block 444 len 20480 22/30 Write block 172 len 20480 data: 10 9 10 10 11 23/30 Read block 531 len 28672 24/30 Read block 441 len 40960 25/30 Read block 66 len 8192 26/30 Read block 11 len 20480 27/30 Write block 357 len 36864 data: 8 10 10 8 10 10 10 7 8 28/30 Read block 36 len 32768 29/30 Write block 52 len 8192 data: 13 14 30/30 Write block 9 len 28672 data: 11 10 12 8 9 8 11 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 9 Done Done Done false 1004 Acked Read 6 Done Done Done false 1005 Acked Write 9 Done Done Done false 1006 Acked Write 1 Done Done Done false 1007 Acked Read 7 Done Done Done false 1008 Acked Write 7 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Write 7 Done Done Done false 1011 Acked Read 6 Done Done Done false 1012 Acked Write 5 Done Done Done false 1013 Acked Write 2 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Write 10 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 2 Done Sent Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 1 4 Done 26 25 26 77 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1002 1002 1002 Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:54:54.053172169Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1707} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 129 129 129 244 244 244 T T T 1 020-039 129 129 129 244 244 244 T T T 2 040-059 118 118 118 221 221 221 T T T 3 060-079 126 126 126 240 240 240 T T T 4 080-099 125 125 125 235 235 235 F F F 5 100-119 125 125 125 235 235 235 T T T 6 120-139 129 129 129 244 244 244 T T T 7 140-159 130 130 130 245 245 245 F F F 8 160-179 120 120 120 224 224 224 T T T 9 180-199 128 128 128 243 243 243 F F F 10 200-219 126 126 126 236 236 236 T T T 11 220-239 129 129 129 244 244 244 F F F 12 240-259 128 128 128 243 243 243 T T T 13 260-279 116 116 116 217 217 217 T T T 14 280-299 123 123 123 231 231 231 T T T 15 300-319 129 129 129 244 244 244 F F F 16 320-339 124 124 124 234 234 234 F F F 17 340-359 119 119 119 223 223 223 T T T 18 360-379 126 126 126 239 239 239 T T T 19 380-399 128 128 128 243 243 243 F F F 20 400-419 124 124 124 232 232 232 F F F 21 420-439 128 128 128 243 243 243 F F F 22 440-459 126 126 126 239 239 239 F F F 23 460-479 128 128 128 242 242 242 F F F 24 480-499 126 126 126 237 237 237 F F F 25 500-519 122 122 122 228 228 228 T T T 26 520-539 120 120 120 224 224 224 T T T 27 540-559 126 126 126 237 237 237 F F F 28 560-579 129 129 129 244 244 244 F F F 29 580-599 127 127 127 241 241 241 F F F Max gen: 130, Max flush: 245 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 131 131 131 246 246 246 F F F 1 020-039 131 131 131 246 246 246 F F F 2 040-059 131 131 131 246 246 246 F F F 3 060-079 131 131 131 246 246 246 F F F 4 080-099 125 125 125 235 235 235 F F F 5 100-119 131 131 131 246 246 246 F F F 6 120-139 131 131 131 246 246 246 F F F 7 140-159 130 130 130 245 245 245 F F F 8 160-179 131 131 131 246 246 246 F F F 9 180-199 128 128 128 243 243 243 F F F 10 200-219 131 131 131 246 246 246 F F F 11 220-239 129 129 129 244 244 244 F F F 12 240-259 131 131 131 246 246 246 F F F 13 260-279 131 131 131 246 246 246 F F F 14 280-299 131 131 131 246 246 246 F F F 15 300-319 129 129 129 244 244 244 F F F 16 320-339 124 124 124 234 234 234 F F F 17 340-359 131 131 131 246 246 246 F F F 18 360-379 131 131 131 246 246 246 F F F 19 380-399 128 128 128 243 243 243 F F F 20 400-419 124 124 124 232 232 232 F F F 21 420-439 128 128 128 243 243 243 F F F 22 440-459 126 126 126 239 239 239 F F F 23 460-479 128 128 128 242 242 242 F F F 24 480-499 126 126 126 237 237 237 F F F 25 500-519 131 131 131 246 246 246 F F F 26 520-539 131 131 131 246 246 246 F F F 27 540-559 126 126 126 237 237 237 F F F 28 560-579 129 129 129 244 244 244 F F F 29 580-599 127 127 127 241 241 241 F F F Max gen: 131, Max flush: 246 Begin loop 66 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.371603219Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.41544408Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.416044873Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.416081974Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.416096604Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.416121505Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715} {"msg":"Crucible 2825634a-b75a-4b98-9231-b4c225cc1a4a has session id: 76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.416567484Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"Upstairs opts: Upstairs UUID: 2825634a-b75a-4b98-9231-b4c225cc1a4a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.416617585Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"Crucible stats registered with UUID: 2825634a-b75a-4b98-9231-b4c225cc1a4a","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.416644846Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.44726448Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.447305841Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715} {"msg":"2825634a-b75a-4b98-9231-b4c225cc1a4a active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.447338792Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.447370752Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"client":"0","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.447418124Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"client":"0","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.447451514Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"client":"1","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.447480425Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"client":"1","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.447511386Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"client":"2","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.447539476Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"client":"2","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.447560007Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.447587067Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.447616188Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"ds_connection connected from Ok(127.0.0.1:50675)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.447756611Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"ds_connection connected from Ok(127.0.0.1:47205)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.447790382Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"ds_connection connected from Ok(127.0.0.1:60367)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.447813412Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.448240981Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"client":"0","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.448268822Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"client":"0","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.448283752Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"client":"2","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.448301723Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"client":"2","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.448405175Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"client":"1","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.448436206Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"client":"1","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.448898356Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"client":"0","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.448924746Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"client":"2","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.448940357Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"client":"1","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.449492778Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"client":"0","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.449535599Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"client":"0","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.44955316Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.44956656Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"client":"2","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.44958058Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"client":"2","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.449594091Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.449606611Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"client":"1","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.449620251Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"client":"1","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"[0]R flush_numbers[0..12]: [246, 246, 246, 246, 235, 246, 246, 245, 246, 243, 246, 244]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.449634292Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"":"downstairs","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"[0]R generation[0..12]: [131, 131, 131, 131, 125, 131, 131, 130, 131, 128, 131, 129]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.449651242Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"":"downstairs","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.449665832Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"":"downstairs","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"[1]R flush_numbers[0..12]: [246, 246, 246, 246, 235, 246, 246, 245, 246, 243, 246, 244]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.449679913Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"":"downstairs","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"[1]R generation[0..12]: [131, 131, 131, 131, 125, 131, 131, 130, 131, 128, 131, 129]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.449693743Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"":"downstairs","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.449707953Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"":"downstairs","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"[2]R flush_numbers[0..12]: [246, 246, 246, 246, 235, 246, 246, 245, 246, 243, 246, 244]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.449721963Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"":"downstairs","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"[2]R generation[0..12]: [131, 131, 131, 131, 125, 131, 131, 130, 131, 128, 131, 129]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.449735794Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"":"downstairs","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.449750094Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"":"downstairs","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.449763224Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"":"downstairs","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.449776275Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"Next flush: 247","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.449799105Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"":"downstairs","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.449812376Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"":"downstairs","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.449825536Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.449837666Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"client":"0","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.449852666Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"client":"1","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.449865907Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"client":"2","":"downstairs","session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.449879417Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.449891817Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"2825634a-b75a-4b98-9231-b4c225cc1a4a is now active with session: 76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.449904767Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.449916888Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715,"session_id":"76f9e4b7-ebc7-47ac-b3d1-7cf82f15158c"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:54:58.449929248Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 528 len 24576 data: 14 15 13 10 9 7 02/30 Write block 359 len 36864 data: 11 9 11 11 11 8 9 10 11 03/30 Flush 04/30 Write block 335 len 32768 data: 13 13 12 14 11 11 10 10 05/30 Flush 06/30 Flush 07/30 Flush 08/30 Write block 134 len 16384 data: 10 11 9 8 09/30 Read block 491 len 24576 10/30 Read block 244 len 16384 11/30 Read block 17 len 16384 12/30 Flush 13/30 Write block 304 len 36864 data: 7 7 6 6 6 4 4 5 5 14/30 Write block 550 len 24576 data: 11 11 14 13 12 13 15/30 Read block 181 len 20480 16/30 Read block 259 len 16384 17/30 Read block 447 len 24576 18/30 Write block 504 len 4096 data: 12 19/30 Flush 20/30 Write block 314 len 20480 data: 10 9 6 5 5 21/30 Write block 74 len 12288 data: 18 17 13 22/30 Flush 23/30 Write block 71 len 20480 data: 17 16 16 19 18 24/30 Read block 491 len 20480 25/30 Read block 258 len 8192 26/30 Write block 232 len 24576 data: 13 15 15 12 13 10 27/30 Write block 505 len 28672 data: 11 12 14 13 15 13 12 28/30 Write block 93 len 40960 data: 12 11 14 10 7 10 9 9 10 11 29/30 Read block 219 len 36864 30/30 Write block 311 len 28672 data: 6 6 6 11 10 7 6 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 5 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:54:58.469234357Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1715} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 131 131 131 246 246 246 F F F 1 020-039 131 131 131 246 246 246 F F F 2 040-059 131 131 131 246 246 246 F F F 3 060-079 132 132 132 253 253 253 T T T 4 080-099 125 125 125 235 235 235 T T T 5 100-119 131 131 131 246 246 246 T T T 6 120-139 132 132 132 251 251 251 F F F 7 140-159 130 130 130 245 245 245 F F F 8 160-179 131 131 131 246 246 246 F F F 9 180-199 128 128 128 243 243 243 F F F 10 200-219 131 131 131 246 246 246 F F F 11 220-239 129 129 129 244 244 244 T T T 12 240-259 131 131 131 246 246 246 F F F 13 260-279 131 131 131 246 246 246 F F F 14 280-299 131 131 131 246 246 246 F F F 15 300-319 132 132 132 253 253 253 T T T 16 320-339 132 132 132 248 248 248 F F F 17 340-359 132 132 132 248 248 248 F F F 18 360-379 132 132 132 247 247 247 F F F 19 380-399 128 128 128 243 243 243 F F F 20 400-419 124 124 124 232 232 232 F F F 21 420-439 128 128 128 243 243 243 F F F 22 440-459 126 126 126 239 239 239 F F F 23 460-479 128 128 128 242 242 242 F F F 24 480-499 126 126 126 237 237 237 F F F 25 500-519 132 132 132 252 252 252 T T T 26 520-539 132 132 132 247 247 247 F F F 27 540-559 132 132 132 252 252 252 F F F 28 560-579 129 129 129 244 244 244 F F F 29 580-599 127 127 127 241 241 241 F F F Max gen: 132, Max flush: 253 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 131 131 131 246 246 246 F F F 1 020-039 131 131 131 246 246 246 F F F 2 040-059 131 131 131 246 246 246 F F F 3 060-079 133 133 133 254 254 254 F F F 4 080-099 133 133 133 254 254 254 F F F 5 100-119 133 133 133 254 254 254 F F F 6 120-139 132 132 132 251 251 251 F F F 7 140-159 130 130 130 245 245 245 F F F 8 160-179 131 131 131 246 246 246 F F F 9 180-199 128 128 128 243 243 243 F F F 10 200-219 131 131 131 246 246 246 F F F 11 220-239 133 133 133 254 254 254 F F F 12 240-259 131 131 131 246 246 246 F F F 13 260-279 131 131 131 246 246 246 F F F 14 280-299 131 131 131 246 246 246 F F F 15 300-319 133 133 133 254 254 254 F F F 16 320-339 132 132 132 248 248 248 F F F 17 340-359 132 132 132 248 248 248 F F F 18 360-379 132 132 132 247 247 247 F F F 19 380-399 128 128 128 243 243 243 F F F 20 400-419 124 124 124 232 232 232 F F F 21 420-439 128 128 128 243 243 243 F F F 22 440-459 126 126 126 239 239 239 F F F 23 460-479 128 128 128 242 242 242 F F F 24 480-499 126 126 126 237 237 237 F F F 25 500-519 133 133 133 254 254 254 F F F 26 520-539 132 132 132 247 247 247 F F F 27 540-559 132 132 132 252 252 252 F F F 28 560-579 129 129 129 244 244 244 F F F 29 580-599 127 127 127 241 241 241 F F F Max gen: 133, Max flush: 254 Begin loop 67 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.739115673Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.783992037Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.784564389Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.78460122Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.7846159Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.78463032Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723} {"msg":"Crucible 7f542c33-724a-4174-9313-9200dcac83d7 has session id: c8470cbb-4121-4416-b4b5-49cc8363e5fd","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.78508557Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"Upstairs opts: Upstairs UUID: 7f542c33-724a-4174-9313-9200dcac83d7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.785131091Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"Crucible stats registered with UUID: 7f542c33-724a-4174-9313-9200dcac83d7","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.785156812Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.815705034Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.815736805Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723} {"msg":"7f542c33-724a-4174-9313-9200dcac83d7 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.815786376Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.815806537Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"client":"0","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.815835587Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"client":"0","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.815853498Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"client":"1","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.815875518Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"client":"1","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.815892969Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"client":"2","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.815910019Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"client":"2","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.815926659Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.81594397Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.81596133Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:35376)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.816082773Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:56564)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.816125124Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:59675)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.816153164Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.816768717Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"client":"0","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.816797398Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"client":"0","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.816824269Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"client":"2","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.816849389Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"client":"2","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.816984152Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"client":"1","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.817019153Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"client":"1","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.817533044Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"client":"2","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.817562665Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"client":"0","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.817579765Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"client":"1","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.818118117Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"client":"1","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.818151078Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"client":"1","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.818166638Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.818179568Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"client":"2","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.818193128Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"client":"2","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.818206249Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.818218349Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"client":"0","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.818231739Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"client":"0","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"[0]R flush_numbers[0..12]: [246, 246, 246, 254, 254, 254, 251, 245, 246, 243, 246, 254]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.81824583Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"":"downstairs","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"[0]R generation[0..12]: [131, 131, 131, 133, 133, 133, 132, 130, 131, 128, 131, 133]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.81826011Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"":"downstairs","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.8182744Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"":"downstairs","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"[1]R flush_numbers[0..12]: [246, 246, 246, 254, 254, 254, 251, 245, 246, 243, 246, 254]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.818288241Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"":"downstairs","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"[1]R generation[0..12]: [131, 131, 131, 133, 133, 133, 132, 130, 131, 128, 131, 133]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.818305951Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"":"downstairs","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.818320761Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"":"downstairs","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"[2]R flush_numbers[0..12]: [246, 246, 246, 254, 254, 254, 251, 245, 246, 243, 246, 254]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.818335002Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"":"downstairs","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"[2]R generation[0..12]: [131, 131, 131, 133, 133, 133, 132, 130, 131, 128, 131, 133]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.818348852Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"":"downstairs","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.818362782Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"":"downstairs","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.818375963Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"":"downstairs","":"downstairs","session_idWait for a query_work_queue command to finish before sending IO ":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.818460544Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"":"downstairs","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"Next flush: 255","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.818486305Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"":"downstairs","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.818500695Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"":"downstairs","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.818520555Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.818543896Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"client":"0","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.818572257Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"client":"1","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.818591567Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"client":"2","":"downstairs","session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.818606848Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.818630508Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"7f542c33-724a-4174-9313-9200dcac83d7 is now active with session: c8470cbb-4121-4416-b4b5-49cc8363e5fd","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.818660319Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.818683919Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723,"session_id":"c8470cbb-4121-4416-b4b5-49cc8363e5fd"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:02.818697479Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 32 len 24576 data: 8 7 8 10 8 9 02/30 Read block 539 len 28672 03/30 Write block 463 len 12288 data: 12 9 7 04/30 Read block 362 len 40960 05/30 Read block 75 len 20480 06/30 Read block 486 len 36864 07/30 Write block 123 len 16384 data: 10 9 8 8 08/30 Write block 370 len 28672 data: 10 9 9 8 9 11 13 09/30 Read block 172 len 12288 10/30 Read block 38 len 4096 11/30 Read block 362 len 16384 12/30 Read block 384 len 24576 13/30 Read block 446 len 40960 14/30 Write block 159 len 40960 data: 8 8 9 9 9 9 6 7 7 9 15/30 Read block 388 len 32768 16/30 Write block 36 len 12288 data: 9 10 12 17/30 Read block 18 len 28672 18/30 Write block 141 len 4096 data: 9 19/30 Write block 389 len 36864 data: 11 10 10 10 11 9 10 10 8 20/30 Write block 36 len 20480 data: 10 11 13 14 13 21/30 Flush 22/30 Write block 376 len 20480 data: 14 14 14 15 14 23/30 Write block 466 len 16384 data: 9 10 11 11 24/30 Read block 20 len 8192 25/30 Write block 140 len 8192 data: 7 10 26/30 Write block 436 len 24576 data: 11 7 8 9 12 14 27/30 Read block 467 len 32768 28/30 Write block 426 len 40960 data: 9 8 10 12 11 11 10 12 11 12 29/30 Read block 507 len 12288 30/30 Write block 401 len 8192 data: 13 11 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 5 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 3 Done Done Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 8 8 7 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:55:02.838320155Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1723} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 131 131 131 246 246 246 F F F 1 020-039 134 134 134 255 255 255 F F F 2 040-059 134 134 134 255 255 255 F F F 3 060-079 133 133 133 254 254 254 F F F 4 080-099 133 133 133 254 254 254 F F F 5 100-119 133 133 133 254 254 254 F F F 6 120-139 134 134 134 255 255 255 F F F 7 140-159 134 134 134 255 255 255 T T T 8 160-179 134 134 134 255 255 255 F F F 9 180-199 128 128 128 243 243 243 F F F 10 200-219 131 131 131 246 246 246 F F F 11 220-239 133 133 133 254 254 254 F F F 12 240-259 131 131 131 246 246 246 F F F 13 260-279 131 131 131 246 246 246 F F F 14 280-299 131 131 131 246 246 246 F F F 15 300-319 133 133 133 254 254 254 F F F 16 320-339 132 132 132 248 248 248 F F F 17 340-359 132 132 132 248 248 248 F F F 18 360-379 134 134 134 255 255 255 T T T 19 380-399 134 134 134 255 255 255 T T T 20 400-419 124 124 124 232 232 232 T T T 21 420-439 128 128 128 243 243 243 T T T 22 440-459 126 126 126 239 239 239 T T T 23 460-479 134 134 134 255 255 255 T T T 24 480-499 126 126 126 237 237 237 F F F 25 500-519 133 133 133 254 254 254 F F F 26 520-539 132 132 132 247 247 247 F F F 27 540-559 132 132 132 252 252 252 F F F 28 560-579 129 129 129 244 244 244 F F F 29 580-599 127 127 127 241 241 241 F F F Max gen: 134, Max flush: 255 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 131 131 131 246 246 246 F F F 1 020-039 134 134 134 255 255 255 F F F 2 040-059 134 134 134 255 255 255 F F F 3 060-079 133 133 133 254 254 254 F F F 4 080-099 133 133 133 254 254 254 F F F 5 100-119 133 133 133 254 254 254 F F F 6 120-139 134 134 134 255 255 255 F F F 7 140-159 135 135 135 256 256 256 F F F 8 160-179 134 134 134 255 255 255 F F F 9 180-199 128 128 128 243 243 243 F F F 10 200-219 131 131 131 246 246 246 F F F 11 220-239 133 133 133 254 254 254 F F F 12 240-259 131 131 131 246 246 246 F F F 13 260-279 131 131 131 246 246 246 F F F 14 280-299 131 131 131 246 246 246 F F F 15 300-319 133 133 133 254 254 254 F F F 16 320-339 132 132 132 248 248 248 F F F 17 340-359 132 132 132 248 248 248 F F F 18 360-379 135 135 135 256 256 256 F F F 19 380-399 135 135 135 256 256 256 F F F 20 400-419 135 135 135 256 256 256 F F F 21 420-439 135 135 135 256 256 256 F F F 22 440-459 135 135 135 256 256 256 F F F 23 460-479 135 135 135 256 256 256 F F F 24 480-499 126 126 126 237 237 237 F F F 25 500-519 133 133 133 254 254 254 F F F 26 520-539 132 132 132 247 247 247 F F F 27 540-559 132 132 132 252 252 252 F F F 28 560-579 129 129 129 244 244 244 F F F 29 580-599 127 127 127 241 241 241 F F F Max gen: 135, Max flush: 256 Begin loop 68 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.11687275Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.160884495Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.161502778Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.161550789Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.161569289Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.16158417Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731} {"msg":"Crucible 2c7f7b39-0d5f-445f-a36c-4e7b7f5d67fa has session id: 4a05eeb7-0e99-40a8-bf7d-f17cf3977294","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.16204011Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"Upstairs opts: Upstairs UUID: 2c7f7b39-0d5f-445f-a36c-4e7b7f5d67fa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.162088181Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"Crucible stats registered with UUID: 2c7f7b39-0d5f-445f-a36c-4e7b7f5d67fa","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.162113731Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.192823798Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.192855878Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731} {"msg":"2c7f7b39-0d5f-445f-a36c-4e7b7f5d67fa active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.192890089Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.192908359Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"client":"0","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.19293811Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"client":"0","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.19295723Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"client":"1","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.192974271Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"client":"1","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.192991031Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"client":"2","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.193016872Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"client":"2","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.193077213Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"":"io task","client":"2","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.193104624Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"":"io task","client":"0","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.193120604Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"":"io task","client":"1","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"ds_connection connected from Ok(127.0.0.1:49469)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.193135524Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"":"io task","client":"2","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"ds_connection connected from Ok(127.0.0.1:47458)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.193237447Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"":"io task","client":"1","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"ds_connection connected from Ok(127.0.0.1:45647)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.193277257Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"":"io task","client":"0","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.193962042Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"client":"1","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.194002853Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"client":"1","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.194035534Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"client":"0","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.194065905Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"client":"0","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.194093565Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"client":"2","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.194116766Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"client":"2","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.194533375Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"client":"1","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.194644377Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"client":"0","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.194835761Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"client":"2","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195083096Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"client":"1","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195108367Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"client":"1","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195122797Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195180969Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"client":"0","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195199889Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"client":"0","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195214059Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195307612Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"client":"2","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195325812Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"client":"2","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"[0]R flush_numbers[0..12]: [246, 255, 255, 254, 254, 254, 255, 256, 255, 243, 246, 254]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195340092Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"":"downstairs","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"[0]R generation[0..12]: [131, 134, 134, 133, 133, 133, 134, 135, 134, 128, 131, 133]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195354042Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"":"downstairs","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195368243Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"":"downstairs","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"[1]R flush_numbers[0..12]: [246, 255, 255, 254, 254, 254, 255, 256, 255, 243, 246, 254]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195382333Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"":"downstairs","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"[1]R generation[0..12]: [131, 134, 134, 133, 133, 133, 134, 135, 134, 128, 131, 133]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195396183Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"":"downstairs","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195410034Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"":"downstairs","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"[2]R flush_numbers[0..12]: [246, 255, 255, 254, 254, 254, 255, 256, 255, 243, 246, 254]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195427474Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"":"downstairs","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"[2]R generation[0..12]: [131, 134, 134, 133, 133, 133, 134, 135, 134, 128, 131, 133]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195441814Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"":"downstairs","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195455905Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"":"downstairs","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195469045Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"":"downstairs","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195482465Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"":"downstairs","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"Next flush: 257","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195495645Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"":"downstairs","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195508746Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195530996Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195543346Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"client":"0","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195556957Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"client":"1","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195570347Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"client":"2","":"downstairs","session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195583838Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195596248Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"2c7f7b39-0d5f-445f-a36c-4e7b7f5d67fa is now active with session: 4a05eeb7-0e99-40a8-bf7d-f17cf3977294","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195609138Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195621368Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731,"session_id":"4a05eeb7-0e99-40a8-bf7d-f17cf3977294"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:07.195635459Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 315 len 4096 data: 11 02/30 Read block 375 len 20480 03/30 Write block 126 len 16384 data: 9 7 12 9 04/30 Read block 29 len 32768 05/30 Write block 454 len 32768 data: 15 12 13 9 9 11 9 10 06/30 Write block 96 len 24576 data: 11 8 11 10 10 11 07/30 Read block 11 len 8192 08/30 Write block 408 len 12288 data: 10 12 15 09/30 Read block 466 len 16384 10/30 Read block 36 len 20480 11/30 Read block 341 len 12288 12/30 Read block 42 len 24576 13/30 Read block 584 len 40960 14/30 Write block 182 len 24576 data: 6 9 12 12 14 13 15/30 Read block 243 len 8192 16/30 Flush 17/30 Write block 109 len 20480 data: 11 13 13 14 14 18/30 Write block 538 len 20480 data: 12 12 9 8 10 19/30 Read block 481 len 24576 20/30 Read block 147 len 24576 21/30 Read block 475 len 28672 22/30 Read block 567 len 8192 23/30 Flush 24/30 Flush 25/30 Write block 580 len 20480 data: 7 7 6 10 10 26/30 Read block 500 len 8192 27/30 Write block 258 len 20480 data: 10 8 7 7 7 28/30 Write block 521 len 24576 data: 10 8 11 15 15 16 29/30 Read block 192 len 36864 30/30 Write block 523 len 32768 data: 12 16 16 17 15 15 16 14 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 5 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:55:07.21508067Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1731} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 131 131 131 246 246 246 F F F 1 020-039 134 134 134 255 255 255 F F F 2 040-059 134 134 134 255 255 255 F F F 3 060-079 133 133 133 254 254 254 F F F 4 080-099 136 136 136 257 257 257 F F F 5 100-119 136 136 136 258 258 258 F F F 6 120-139 136 136 136 257 257 257 F F F 7 140-159 135 135 135 256 256 256 F F F 8 160-179 134 134 134 255 255 255 F F F 9 180-199 136 136 136 257 257 257 F F F 10 200-219 131 131 131 246 246 246 F F F 11 220-239 133 133 133 254 254 254 F F F 12 240-259 131 131 131 246 246 246 T T T 13 260-279 131 131 131 246 246 246 T T T 14 280-299 131 131 131 246 246 246 F F F 15 300-319 136 136 136 257 257 257 F F F 16 320-339 132 132 132 248 248 248 F F F 17 340-359 132 132 132 248 248 248 F F F 18 360-379 135 135 135 256 256 256 F F F 19 380-399 135 135 135 256 256 256 F F F 20 400-419 136 136 136 257 257 257 F F F 21 420-439 135 135 135 256 256 256 F F F 22 440-459 136 136 136 257 257 257 F F F 23 460-479 136 136 136 257 257 257 F F F 24 480-499 126 126 126 237 237 237 F F F 25 500-519 133 133 133 254 254 254 F F F 26 520-539 136 136 136 258 258 258 T T T 27 540-559 136 136 136 258 258 258 F F F 28 560-579 129 129 129 244 244 244 F F F 29 580-599 127 127 127 241 241 241 T T T Max gen: 136, Max flush: 258 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 131 131 131 246 246 246 F F F 1 020-039 134 134 134 255 255 255 F F F 2 040-059 134 134 134 255 255 255 F F F 3 060-079 133 133 133 254 254 254 F F F 4 080-099 136 136 136 257 257 257 F F F 5 100-119 136 136 136 258 258 258 F F F 6 120-139 136 136 136 257 257 257 F F F 7 140-159 135 135 135 256 256 256 F F F 8 160-179 134 134 134 255 255 255 F F F 9 180-199 136 136 136 257 257 257 F F F 10 200-219 131 131 131 246 246 246 F F F 11 220-239 133 133 133 254 254 254 F F F 12 240-259 137 137 137 259 259 259 F F F 13 260-279 137 137 137 259 259 259 F F F 14 280-299 131 131 131 246 246 246 F F F 15 300-319 136 136 136 257 257 257 F F F 16 320-339 132 132 132 248 248 248 F F F 17 340-359 132 132 132 248 248 248 F F F 18 360-379 135 135 135 256 256 256 F F F 19 380-399 135 135 135 256 256 256 F F F 20 400-419 136 136 136 257 257 257 F F F 21 420-439 135 135 135 256 256 256 F F F 22 440-459 136 136 136 257 257 257 F F F 23 460-479 136 136 136 257 257 257 F F F 24 480-499 126 126 126 237 237 237 F F F 25 500-519 133 133 133 254 254 254 F F F 26 520-539 137 137 137 259 259 259 F F F 27 540-559 136 136 136 258 258 258 F F F 28 560-579 129 129 129 244 244 244 F F F 29 580-599 137 137 137 259 259 259 F F F Max gen: 137, Max flush: 259 Begin loop 69 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.471894803Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.516845639Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.517499313Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.517546684Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.517561464Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.517575994Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739} {"msg":"Crucible 0fb05fdb-fff2-4d86-accc-3be69409d37c has session id: a08c2d06-e15f-4b3b-bbac-a1562c701468","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.517942242Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"Upstairs opts: Upstairs UUID: 0fb05fdb-fff2-4d86-accc-3be69409d37c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.517982303Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"Crucible stats registered with UUID: 0fb05fdb-fff2-4d86-accc-3be69409d37c","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.517998744Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.549442816Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.549473966Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739} {"msg":"0fb05fdb-fff2-4d86-accc-3be69409d37c active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.549558798Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.549583059Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"client":"0","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.549611509Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"client":"0","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.54962922Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"client":"1","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.54964614Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"client":"1","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.54966247Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"client":"2","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.549689391Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"client":"2","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.549722882Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.549757542Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.549794273Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"ds_connection connected from Ok(127.0.0.1:46488)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.549857535Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"ds_connection connected from Ok(127.0.0.1:63455)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.550009968Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"ds_connection connected from Ok(127.0.0.1:62515)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.550044679Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.550636321Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"client":"0","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.550677822Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"client":"0","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.550709973Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"client":"1","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.550777755Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"client":"1","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.550848766Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"client":"2","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.550884987Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"client":"2","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.551309976Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"client":"0","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.551333377Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"client":"1","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.551396308Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"client":"2","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.551820247Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"client":"0","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.551852178Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"client":"0","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.551870168Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.55193186Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"client":"1","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.55195998Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"client":"1","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.55197453Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.551987271Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"client":"2","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.552001151Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"client":"2","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"[0]R flush_numbers[0..12]: [246, 255, 255, 254, 257, 258, 257, 256, 255, 257, 246, 254]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.552015502Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"":"downstairs","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"[0]R generation[0..12]: [131, 134, 134, 133, 136, 136, 136, 135, 134, 136, 131, 133]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.552029562Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"":"downstairs","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.552043502Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"":"downstairs","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"[1]R flush_numbers[0..12]: [246, 255, 255, 254, 257, 258, 257, 256, 255, 257, 246, 254]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.552057482Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"":"downstairs","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"[1]R generation[0..12]: [131, 134, 134, 133, 136, 136, 136, 135, 134, 136, 131, 133]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.552071123Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"":"downstairs","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.552085303Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"":"downstairs","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"[2]R flush_numbers[0..12]: [246, 255, 255, 254, 257, 258, 257, 256, 255, 257, 246, 254]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.552099123Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"":"downstairs","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"[2]R generation[0..12]: [131, 134, 134, 133, 136, 136, 136, 135, 134, 136, 131, 133]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.552112944Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"":"downstairs","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.552127034Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"":"downstairs","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.552143954Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"":"downstairs","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.552158025Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"":"downstairs","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"Next flush: 260","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.552171315Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"":"downstairs","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.552184375Z","hostnameWait for a query_work_queue command to finish before sending IO ":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"":"downstairs","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.552205895Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.552218216Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"client":"0","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.552231576Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"client":"1","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.552244876Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"client":"2","":"downstairs","session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.552258177Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.552270667Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"0fb05fdb-fff2-4d86-accc-3be69409d37c is now active with session: a08c2d06-e15f-4b3b-bbac-a1562c701468","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.552283637Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.552296037Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739,"session_id":"a08c2d06-e15f-4b3b-bbac-a1562c701468"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:11.552308568Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 185 len 16384 data: 13 15 14 12 02/30 Read block 151 len 32768 03/30 Read block 589 len 4096 04/30 Read block 344 len 24576 05/30 Write block 403 len 28672 data: 11 12 11 11 9 11 13 06/30 Write block 392 len 24576 data: 11 12 10 11 11 9 07/30 Read block 326 len 36864 08/30 Read block 361 len 4096 09/30 Read block 152 len 24576 10/30 Write block 478 len 24576 data: 12 11 12 11 12 11 11/30 Read block 466 len 40960 12/30 Write block 566 len 36864 data: 4 5 5 9 11 13 13 15 16 13/30 Read block 236 len 36864 14/30 Write block 248 len 28672 data: 17 15 13 12 12 12 12 15/30 Write block 336 len 8192 data: 14 13 16/30 Write block 428 len 24576 data: 11 13 12 12 11 13 17/30 Write block 242 len 8192 data: 9 12 18/30 Write block 422 len 28672 data: 10 9 9 9 10 9 12 19/30 Write block 64 len 32768 data: 9 11 13 16 15 15 13 18 20/30 Write block 85 len 12288 data: 5 5 6 21/30 Read block 471 len 36864 22/30 Write block 237 len 12288 data: 11 9 7 23/30 Read block 426 len 12288 24/30 Write block 65 len 36864 data: 12 14 17 16 16 14 19 17 17 25/30 Write block 159 len 32768 data: 9 9 10 10 10 10 7 8 26/30 Flush 27/30 Write block 81 len 4096 data: 9 28/30 Write block 531 len 16384 data: 11 10 8 8 29/30 Write block 374 len 32768 data: 10 12 15 15 15 16 15 13 30/30 Write block 283 len 8192 data: 8 11 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 1 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:55:11.571594716Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1739} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 131 131 131 246 246 246 F F F 1 020-039 134 134 134 255 255 255 F F F 2 040-059 134 134 134 255 255 255 F F F 3 060-079 138 138 138 260 260 260 F F F 4 080-099 138 138 138 260 260 260 T T T 5 100-119 136 136 136 258 258 258 F F F 6 120-139 136 136 136 257 257 257 F F F 7 140-159 138 138 138 260 260 260 F F F 8 160-179 138 138 138 260 260 260 F F F 9 180-199 138 138 138 260 260 260 F F F 10 200-219 131 131 131 246 246 246 F F F 11 220-239 138 138 138 260 260 260 F F F 12 240-259 138 138 138 260 260 260 F F F 13 260-279 137 137 137 259 259 259 F F F 14 280-299 131 131 131 246 246 246 T T T 15 300-319 136 136 136 257 257 257 F F F 16 320-339 138 138 138 260 260 260 F F F 17 340-359 132 132 132 248 248 248 F F F 18 360-379 135 135 135 256 256 256 T T T 19 380-399 138 138 138 260 260 260 T T T 20 400-419 138 138 138 260 260 260 F F F 21 420-439 138 138 138 260 260 260 F F F 22 440-459 136 136 136 257 257 257 F F F 23 460-479 138 138 138 260 260 260 F F F 24 480-499 138 138 138 260 260 260 F F F 25 500-519 133 133 133 254 254 254 F F F 26 520-539 137 137 137 259 259 259 T T T 27 540-559 136 136 136 258 258 258 F F F 28 560-579 138 138 138 260 260 260 F F F 29 580-599 137 137 137 259 259 259 F F F Max gen: 138, Max flush: 260 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 131 131 131 246 246 246 F F F 1 020-039 134 134 134 255 255 255 F F F 2 040-059 134 134 134 255 255 255 F F F 3 060-079 138 138 138 260 260 260 F F F 4 080-099 139 139 139 261 261 261 F F F 5 100-119 136 136 136 258 258 258 F F F 6 120-139 136 136 136 257 257 257 F F F 7 140-159 138 138 138 260 260 260 F F F 8 160-179 138 138 138 260 260 260 F F F 9 180-199 138 138 138 260 260 260 F F F 10 200-219 131 131 131 246 246 246 F F F 11 220-239 138 138 138 260 260 260 F F F 12 240-259 138 138 138 260 260 260 F F F 13 260-279 137 137 137 259 259 259 F F F 14 280-299 139 139 139 261 261 261 F F F 15 300-319 136 136 136 257 257 257 F F F 16 320-339 138 138 138 260 260 260 F F F 17 340-359 132 132 132 248 248 248 F F F 18 360-379 139 139 139 261 261 261 F F F 19 380-399 139 139 139 261 261 261 F F F 20 400-419 138 138 138 260 260 260 F F F 21 420-439 138 138 138 260 260 260 F F F 22 440-459 136 136 136 257 257 257 F F F 23 460-479 138 138 138 260 260 260 F F F 24 480-499 138 138 138 260 260 260 F F F 25 500-519 133 133 133 254 254 254 F F F 26 520-539 139 139 139 261 261 261 F F F 27 540-559 136 136 136 258 258 258 F F F 28 560-579 138 138 138 260 260 260 F F F 29 580-599 137 137 137 259 259 259 F F F Max gen: 139, Max flush: 261 Begin loop 70 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.835703667Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.879252072Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.879857055Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.879894236Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.879908906Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.879923627Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747} {"msg":"Crucible 03571389-c786-42e0-a629-6ed117e075a8 has session id: a838c380-9c02-41b5-8eb2-fe077f50d00a","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.880325785Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"Upstairs opts: Upstairs UUID: 03571389-c786-42e0-a629-6ed117e075a8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.880368426Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"Crucible stats registered with UUID: 03571389-c786-42e0-a629-6ed117e075a8","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.880396157Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.911960592Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.911995382Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747} {"msg":"03571389-c786-42e0-a629-6ed117e075a8 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.912030803Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.912048413Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"client":"0","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.912077344Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"client":"0","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.912095074Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"client":"1","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.912112105Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"client":"1","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.912192727Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"client":"2","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.912230928Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"client":"2","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.912263418Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.912289919Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.912316919Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"ds_connection connected from Ok(127.0.0.1:56154)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.91235186Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"ds_connection connected from Ok(127.0.0.1:59530)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.912522854Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"ds_connection connected from Ok(127.0.0.1:50657)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.912553235Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.913013174Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"client":"1","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.913055645Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"client":"1","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.913088746Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"client":"0","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.913116727Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"client":"0","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.913143387Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"client":"2","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.913171528Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"client":"2","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.91372057Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"client":"1","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.913765191Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"client":"0","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.913803372Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"client":"2","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.914276772Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"client":"1","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.914307682Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"client":"1","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.914324843Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.914339233Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"client":"0","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.914354554Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"client":"0","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.914369254Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.914382584Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"client":"2","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.914400984Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"client":"2","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"[0]R flush_numbers[0..12]: [246, 255, 255, 260, 261, 258, 257, 260, 260, 260, 246, 260]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.914416625Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"":"downstairs","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"[0]R generation[0..12]: [131, 134, 134, 138, 139, 136, 136, 138, 138, 138, 131, 138]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.914431195Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"":"downstairs","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.914446505Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"":"downstairs","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"[1]R flush_numbers[0..12]: [246, 255, 255, 260, 261, 258, 257, 260, 260, 260, 246, 260]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.914462046Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"":"downstairs","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"[1]R generation[0..12]: [131, 134, 134, 138, 139, 136, 136, 138, 138, 138, 131, 138]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.914477406Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"":"downstairs","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.914493226Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"":"downstairs","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"[2]R flush_numbers[0..12]: [246, 255, 255, 260, 261, 258, 257, 260, 260, 260, 246, 260]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.914508217Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"":"downstairs","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"[2]R generation[0..12]: [131, 134, 134, 138, 139, 136, 136, 138, 138, 138, 131, 138]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.914523307Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"":"downstairs","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2024-11-28T04:55:15.914549038Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"":"downstairs","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.914563768Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"":"downstairs","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.914578268Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"":"downstairs","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"Next flush: 262","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.914592539Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"":"downstairs","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.914609329Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"":"downstairs","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.914624589Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.91463793Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"client":"0","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.91465256Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"client":"1","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.91466734Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 1747,"client":"2","":"downstairs","session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.914685701Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.914699901Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"03571389-c786-42e0-a629-6ed117e075a8 is now active with session: a838c380-9c02-41b5-8eb2-fe077f50d00a","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.914713861Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.914727062Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747,"session_id":"a838c380-9c02-41b5-8eb2-fe077f50d00a"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:15.914740542Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 306 len 32768 data: 7 7 7 5 5 7 7 7 02/30 Flush 03/30 Write block 112 len 28672 data: 15 15 14 14 12 12 11 04/30 Write block 582 len 8192 data: 7 11 05/30 Write block 51 len 40960 data: 13 14 15 14 14 15 15 15 13 13 06/30 Read block 175 len 40960 07/30 Read block 313 len 40960 08/30 Write block 547 len 28672 data: 15 15 12 12 12 15 14 09/30 Write block 293 len 40960 data: 5 6 5 6 7 9 10 10 9 9 10/30 Write block 356 len 28672 data: 8 9 11 12 10 12 12 11/30 Read block 82 len 8192 12/30 Read block 110 len 28672 13/30 Flush 14/30 Write block 95 len 12288 data: 15 12 9 15/30 Write block 53 len 12288 data: 16 15 15 16/30 Read block 383 len 20480 17/30 Read block 532 len 28672 18/30 Write block 391 len 24576 data: 11 12 13 11 12 12 19/30 Read block 471 len 4096 20/30 Write block 62 len 24576 data: 10 9 10 13 15 18 21/30 Read block 590 len 8192 22/30 Read block 62 len 36864 23/30 Read block 378 len 24576 24/30 Write block 235 len 16384 data: 13 14 12 10 25/30 Read block 273 len 8192 26/30 Write block 591 len 20480 data: 10 9 7 6 5 27/30 Read block 497 len 20480 28/30 Read block 372 len 24576 29/30 Write block 26 len 4096 data: 14 30/30 Read block 208 len 28672 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 3 Done Done Done false 1014 Acked Write 3 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 1 Sent Done Done false 1029 Acked Read 7 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 0 2 Done 15 17 17 49 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:55:15.93446626Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1747} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 131 131 131 246 246 246 F F F 1 020-039 134 134 134 255 255 255 T T T 2 040-059 140 140 140 263 263 263 T T T 3 060-079 140 140 140 263 263 263 T T T 4 080-099 139 139 139 261 261 261 T T T 5 100-119 140 140 140 263 263 263 F F F 6 120-139 136 136 136 257 257 257 F F F 7 140-159 138 138 138 260 260 260 F F F 8 160-179 138 138 138 260 260 260 F F F 9 180-199 138 138 138 260 260 260 F F F 10 200-219 131 131 131 246 246 246 F F F 11 220-239 138 138 138 260 260 260 T T T 12 240-259 138 138 138 260 260 260 F F F 13 260-279 137 137 137 259 259 259 F F F 14 280-299 140 140 140 263 263 263 F F F 15 300-319 140 140 140 263 263 263 F F F 16 320-339 138 138 138 260 260 260 F F F 17 340-359 140 140 140 263 263 263 F F F 18 360-379 140 140 140 263 263 263 F F F 19 380-399 139 139 139 261 261 261 T T T 20 400-419 138 138 138 260 260 260 F F F 21 420-439 138 138 138 260 260 260 F F F 22 440-459 136 136 136 257 257 257 F F F 23 460-479 138 138 138 260 260 260 F F F 24 480-499 138 138 138 260 260 260 F F F 25 500-519 133 133 133 254 254 254 F F F 26 520-539 139 139 139 261 261 261 F F F 27 540-559 140 140 140 263 263 263 F F F 28 560-579 138 138 138 260 260 260 F F F 29 580-599 140 140 140 263 263 263 T T T Max gen: 140, Max flush: 263 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 131 131 131 246 246 246 F F F 1 020-039 141 141 141 264 264 264 F F F 2 040-059 141 141 141 264 264 264 F F F 3 060-079 141 141 141 264 264 264 F F F 4 080-099 141 141 141 264 264 264 F F F 5 100-119 140 140 140 263 263 263 F F F 6 120-139 136 136 136 257 257 257 F F F 7 140-159 138 138 138 260 260 260 F F F 8 160-179 138 138 138 260 260 260 F F F 9 180-199 138 138 138 260 260 260 F F F 10 200-219 131 131 131 246 246 246 F F F 11 220-239 141 141 141 264 264 264 F F F 12 240-259 138 138 138 260 260 260 F F F 13 260-279 137 137 137 259 259 259 F F F 14 280-299 140 140 140 263 263 263 F F F 15 300-319 140 140 140 263 263 263 F F F 16 320-339 138 138 138 260 260 260 F F F 17 340-359 140 140 140 263 263 263 F F F 18 360-379 140 140 140 263 263 263 F F F 19 380-399 141 141 141 264 264 264 F F F 20 400-419 138 138 138 260 260 260 F F F 21 420-439 138 138 138 260 260 260 F F F 22 440-459 136 136 136 257 257 257 F F F 23 460-479 138 138 138 260 260 260 F F F 24 480-499 138 138 138 260 260 260 F F F 25 500-519 133 133 133 254 254 254 F F F 26 520-539 139 139 139 261 261 261 F F F 27 540-559 140 140 140 263 263 263 F F F 28 560-579 138 138 138 260 260 260 F F F 29 580-599 141 141 141 264 264 264 F F F Max gen: 141, Max flush: 264 Begin loop 71 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.211678855Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:55:20.254154897Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.254506285Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.285972817Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.286510209Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.28654246Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.28655636Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.28657048Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755} {"msg":"Crucible 1cd6f0d5-4a4a-48a2-8241-a0745da5a2a7 has session id: ce08c38c-324a-4664-a9e5-8d0981ae4d32","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.286986599Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"Upstairs opts: Upstairs UUID: 1cd6f0d5-4a4a-48a2-8241-a0745da5a2a7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.28703218Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"Crucible stats registered with UUID: 1cd6f0d5-4a4a-48a2-8241-a0745da5a2a7","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.287060591Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.318491653Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.318536043Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755} {"msg":"1cd6f0d5-4a4a-48a2-8241-a0745da5a2a7 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.318602155Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.318632245Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"client":"0","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.318658526Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"client":"0","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.318675177Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"client":"1","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.318691487Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"client":"1","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.318708707Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"client":"2","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.318734348Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"client":"2","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.318758758Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"":"io task","client":"2","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.318783769Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"":"io task","client":"0","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.318809549Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"":"io task","client":"1","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"ds_connection connected from Ok(127.0.0.1:38661)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.31883603Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"":"io task","client":"2","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"ds_connection connected from Ok(127.0.0.1:42817)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.31886205Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"":"io task","client":"1","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"ds_connection connected from Ok(127.0.0.1:56849)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.319100296Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"":"io task","client":"0","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.319615997Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"client":"1","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.319643807Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"client":"1","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.319659168Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"client":"2","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.319680118Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"client":"2","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.319706439Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"client":"0","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.31973214Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"client":"0","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.32022321Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"client":"1","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.320256241Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"client":"0","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.320358983Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"client":"2","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.320773112Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"client":"1","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.320792503Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"client":"1","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.320806233Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.320896485Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"client":"0","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.320919555Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"client":"0","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.320933686Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.320949986Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"client":"2","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.320963686Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"client":"2","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"[0]R flush_numbers[0..12]: [246, 264, 264, 264, 264, 263, 257, 260, 260, 260, 246, 264]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.320977336Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"":"downstairs","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"[0]R generation[0..12]: [131, 141, 141, 141, 141, 140, 136, 138, 138, 138, 131, 141]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.320991437Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"":"downstairs","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.321006717Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"":"downstairs","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"[1]R flush_numbers[0..12]: [246, 264, 264, 264, 264, 263, 257, 260, 260, 260, 246, 264]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.321020707Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"":"downstairs","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"[1]R generation[0..12]: [131, 141, 141, 141, 141, 140, 136, 138, 138, 138, 131, 141]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.321034528Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"":"downstairs","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.321048348Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"":"downstairs","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"[2]R flush_numbers[0..12]: [246, 264, 264, 264, 264, 263, 257, 260, 260, 260, 246, 264]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.321062108Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"":"downstairs","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"[2]R generation[0..12]: [131, 141, 141, 141, 141, 140, 136, 138, 138, 138, 131, 141]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.321075758Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"":"downstairs","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.321089679Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"":"downstairs","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"Max found gen is 142","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2024-11-28T04:55:20.321110249Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"":"downstairs","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.32112377Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"":"downstairs","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"Next flush: 265","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.32113683Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"":"downstairs","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.32115187Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"":"downstairs","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.321165231Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.321177361Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"client":"0","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.321190871Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"client":"1","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.321204301Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"client":"2","":"downstairs","session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.321217842Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.321230302Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"1cd6f0d5-4a4a-48a2-8241-a0745da5a2a7 is now active with session: ce08c38c-324a-4664-a9e5-8d0981ae4d32","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.321243012Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.321255033Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755,"session_id":"ce08c38c-324a-4664-a9e5-8d0981ae4d32"} {"msg":"The guest has finished waiting for activation with:142"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:20.321270733Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 290 len 16384 data: 8 5 7 6 02/30 Flush 03/30 Write block 45 len 20480 data: 11 11 9 10 10 04/30 Write block 125 len 36864 data: 9 10 8 13 10 11 12 12 10 05/30 Read block 320 len 40960 06/30 Write block 178 len 28672 data: 6 6 5 5 7 10 13 07/30 Read block 76 len 8192 08/30 Write block 117 len 32768 data: 13 12 9 7 9 7 11 10 09/30 Read block 443 len 12288 10/30 Read block 519 len 32768 11/30 Read block 551 len 40960 12/30 Flush 13/30 Write block 272 len 28672 data: 15 12 12 11 10 10 10 14/30 Read block 136 len 20480 15/30 Write block 417 len 32768 data: 6 3 5 7 9 11 10 10 16/30 Read block 443 len 28672 17/30 Read block 197 len 20480 18/30 Write block 485 len 24576 data: 11 10 10 8 7 10 19/30 Read block 334 len 20480 20/30 Write block 327 len 4096 data: 9 21/30 Read block 262 len 36864 22/30 Read block 395 len 40960 23/30 Read block 41 len 32768 24/30 Read block 393 len 12288 25/30 Flush 26/30 Write block 376 len 20480 data: 16 16 16 17 16 27/30 Write block 26 len 20480 data: 15 14 14 11 7 28/30 Read block 65 len 36864 29/30 Write block 559 len 12288 data: 10 8 5 30/30 Read block 449 len 16384 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 5 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 4 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 2 2 Done 5 5 3 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:55:20.342244188Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1755} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 131 131 131 246 246 246 F F F 1 020-039 141 141 141 264 264 264 T T T 2 040-059 142 142 142 266 266 266 F F F 3 060-079 141 141 141 264 264 264 F F F 4 080-099 141 141 141 264 264 264 F F F 5 100-119 142 142 142 266 266 266 F F F 6 120-139 142 142 142 266 266 266 F F F 7 140-159 138 138 138 260 260 260 F F F 8 160-179 142 142 142 266 266 266 F F F 9 180-199 142 142 142 266 266 266 F F F 10 200-219 131 131 131 246 246 246 F F F 11 220-239 141 141 141 264 264 264 F F F 12 240-259 138 138 138 260 260 260 F F F 13 260-279 142 142 142 267 267 267 F F F 14 280-299 142 142 142 265 265 265 F F F 15 300-319 140 140 140 263 263 263 F F F 16 320-339 142 142 142 267 267 267 F F F 17 340-359 140 140 140 263 263 263 F F F 18 360-379 140 140 140 263 263 263 T T T 19 380-399 141 141 141 264 264 264 T T T 20 400-419 142 142 142 267 267 267 F F F 21 420-439 142 142 142 267 267 267 F F F 22 440-459 136 136 136 257 257 257 F F F 23 460-479 138 138 138 260 260 260 F F F 24 480-499 142 142 142 267 267 267 F F F 25 500-519 133 133 133 254 254 254 F F F 26 520-539 139 139 139 261 261 261 F F F 27 540-559 140 140 140 263 263 263 T T T 28 560-579 138 138 138 260 260 260 T T T 29 580-599 141 141 141 264 264 264 F F F Max gen: 142, Max flush: 267 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 131 131 131 246 246 246 F F F 1 020-039 143 143 143 268 268 268 F F F 2 040-059 142 142 142 266 266 266 F F F 3 060-079 141 141 141 264 264 264 F F F 4 080-099 141 141 141 264 264 264 F F F 5 100-119 142 142 142 266 266 266 F F F 6 120-139 142 142 142 266 266 266 F F F 7 140-159 138 138 138 260 260 260 F F F 8 160-179 142 142 142 266 266 266 F F F 9 180-199 142 142 142 266 266 266 F F F 10 200-219 131 131 131 246 246 246 F F F 11 220-239 141 141 141 264 264 264 F F F 12 240-259 138 138 138 260 260 260 F F F 13 260-279 142 142 142 267 267 267 F F F 14 280-299 142 142 142 265 265 265 F F F 15 300-319 140 140 140 263 263 263 F F F 16 320-339 142 142 142 267 267 267 F F F 17 340-359 140 140 140 263 263 263 F F F 18 360-379 143 143 143 268 268 268 F F F 19 380-399 143 143 143 268 268 268 F F F 20 400-419 142 142 142 267 267 267 F F F 21 420-439 142 142 142 267 267 267 F F F 22 440-459 136 136 136 257 257 257 F F F 23 460-479 138 138 138 260 260 260 F F F 24 480-499 142 142 142 267 267 267 F F F 25 500-519 133 133 133 254 254 254 F F F 26 520-539 139 139 139 261 261 261 F F F 27 540-559 143 143 143 268 268 268 F F F 28 560-579 143 143 143 268 268 268 F F F 29 580-599 141 141 141 264 264 264 F F F Max gen: 143, Max flush: 268 Begin loop 72 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.638861224Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:55:24.681627352Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.6820037Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.713722988Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.71427413Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.714309471Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.714323712Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.714338032Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763} {"msg":"Crucible a089b387-fcdf-4930-b667-470c5784ff46 has session id: 096049b8-6eaa-4ca2-83e4-d050a0888430","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.714810092Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"Upstairs opts: Upstairs UUID: a089b387-fcdf-4930-b667-470c5784ff46, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.714854983Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"Crucible stats registered with UUID: a089b387-fcdf-4930-b667-470c5784ff46","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.714879524Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.745640161Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.745674652Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763} {"msg":"a089b387-fcdf-4930-b667-470c5784ff46 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.745711272Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.745729383Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"client":"0","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.745756983Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"client":"0","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.745774684Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"client":"1","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.745791414Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"client":"1","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.745807575Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"client":"2","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.745830245Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"client":"2","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.745861316Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"":"io task","client":"2","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.745896126Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"":"io task","client":"0","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.745931727Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"":"io task","client":"1","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"ds_connection connected from Ok(127.0.0.1:63256)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.745969128Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"":"io task","client":"2","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"ds_connection connected from Ok(127.0.0.1:45853)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.746015089Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"":"io task","client":"1","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"ds_connection connected from Ok(127.0.0.1:61890)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.746122331Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"":"io task","client":"0","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.746721814Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"client":"2","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.746765035Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"client":"2","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.746843187Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"client":"1","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.746877648Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"client":"1","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.746894078Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"client":"0","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.746907828Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"client":"0","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.74743053Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"client":"1","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.74746167Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"client":"2","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.747478941Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"client":"0","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.748011452Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"client":"1","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.748040363Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"client":"1","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.748055743Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.748068424Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"client":"2","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.748082114Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"client":"2","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.748095604Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.748108234Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"client":"0","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.748121945Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"client":"0","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"[0]R flush_numbers[0..12]: [246, 268, 266, 264, 264, 266, 266, 260, 266, 266, 246, 264]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.748140195Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"":"downstairs","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"[0]R generation[0..12]: [131, 143, 142, 141, 141, 142, 142, 138, 142, 142, 131, 141]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.748154845Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"":"downstairs","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.748169056Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"":"downstairs","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"[1]R flush_numbers[0..12]: [246, 268, 266, 264, 264, 266, 266, 260, 266, 266, 246, 264]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.748182916Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"":"downstairs","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"[1]R generation[0..12]: [131, 143, 142, 141, 141, 142, 142, 138, 142, 142, 131, 141]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.748196656Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"":"downstairs","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.748210647Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"":"downstairs","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"[2]R flush_numbers[0..12]: [246, 268, 266, 264, 264, 266, 266, 260, 266, 266, 246, 264]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.748224577Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"":"downstairs","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"[2]R generation[0..12]: [131, 143, 142, 141, 141, 142, 142, 138, 142, 142, 131, 141]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.748238337Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"":"downstairs","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.748252197Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"":"downstairs","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.748266068Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"":"downstairs","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.748288298Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"":"downstairs","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"Next flush: 269","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.748301478Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"":"downstairs","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.748314499Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"":"downstairs","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.748328059Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.748341749Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"client":"0","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.74835517Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"client":"1","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.7483684Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"client":"2","":"downstairs","session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.74838158Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.74839406Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"a089b387-fcdf-4930-b667-470c5784ff46 is now active with session: 096049b8-6eaa-4ca2-83e4-d050a0888430","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.748406811Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.748418981Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763,"session_id":"096049b8-6eaa-4ca2-83e4-d050a0888430"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:24.748431231Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 201 len 12288 data: 12 14 12 02/30 Read block 229 len 36864 03/30 Read block 526 len 40960 04/30 Read block 344 len 24576 05/30 Write block 173 len 24576 data: 10 11 11 12 8 7 06/30 Flush 07/30 Write block 501 len 20480 data: 15 10 12 13 12 08/30 Flush 09/30 Write block 335 len 8192 data: 14 15 10/30 Write block 305 len 12288 data: 8 8 8 11/30 Write block 80 len 28672 data: 8 10 8 4 4 6 6 12/30 Write block 269 len 40960 data: 14 17 15 16 13 13 12 11 11 11 13/30 Read block 437 len 40960 14/30 Read block 560 len 32768 15/30 Flush 16/30 Write block 19 len 36864 data: 16 14 16 14 15 15 13 16 15 17/30 Read block 213 len 8192 18/30 Write block 421 len 32768 data: 10 12 11 11 10 11 10 13 19/30 Read block 289 len 16384 20/30 Write block 336 len 12288 data: 16 14 15 21/30 Write block 97 len 4096 data: 10 22/30 Write block 269 len 12288 data: 15 18 16 23/30 Flush 24/30 Write block 566 len 8192 data: 5 6 25/30 Write block 182 len 36864 data: 8 11 14 14 16 15 13 14 13 26/30 Read block 316 len 8192 27/30 Write block 319 len 36864 data: 6 5 6 6 8 7 8 8 10 28/30 Write block 174 len 28672 data: 12 12 13 9 8 7 6 29/30 Write block 530 len 20480 data: 15 12 11 9 9 30/30 Write block 570 len 28672 data: 12 14 14 16 17 16 11 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 2 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:55:24.767656658Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1763} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 272 272 272 F F F 1 020-039 144 144 144 272 272 272 F F F 2 040-059 142 142 142 266 266 266 F F F 3 060-079 141 141 141 264 264 264 F F F 4 080-099 144 144 144 272 272 272 F F F 5 100-119 142 142 142 266 266 266 F F F 6 120-139 142 142 142 266 266 266 F F F 7 140-159 138 138 138 260 260 260 F F F 8 160-179 144 144 144 269 269 269 T T T 9 180-199 142 142 142 266 266 266 T T T 10 200-219 144 144 144 269 269 269 F F F 11 220-239 141 141 141 264 264 264 F F F 12 240-259 138 138 138 260 260 260 F F F 13 260-279 144 144 144 272 272 272 F F F 14 280-299 142 142 142 265 265 265 F F F 15 300-319 144 144 144 271 271 271 T T T 16 320-339 144 144 144 272 272 272 T T T 17 340-359 140 140 140 263 263 263 F F F 18 360-379 143 143 143 268 268 268 F F F 19 380-399 143 143 143 268 268 268 F F F 20 400-419 142 142 142 267 267 267 F F F 21 420-439 144 144 144 272 272 272 F F F 22 440-459 136 136 136 257 257 257 F F F 23 460-479 138 138 138 260 260 260 F F F 24 480-499 142 142 142 267 267 267 F F F 25 500-519 144 144 144 270 270 270 F F F 26 520-539 139 139 139 261 261 261 T T T 27 540-559 143 143 143 268 268 268 F F F 28 560-579 143 143 143 268 268 268 T T T 29 580-599 141 141 141 264 264 264 F F F Max gen: 144, Max flush: 272 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 272 272 272 F F F 1 020-039 144 144 144 272 272 272 F F F 2 040-059 142 142 142 266 266 266 F F F 3 060-079 141 141 141 264 264 264 F F F 4 080-099 144 144 144 272 272 272 F F F 5 100-119 142 142 142 266 266 266 F F F 6 120-139 142 142 142 266 266 266 F F F 7 140-159 138 138 138 260 260 260 F F F 8 160-179 145 145 145 273 273 273 F F F 9 180-199 145 145 145 273 273 273 F F F 10 200-219 144 144 144 269 269 269 F F F 11 220-239 141 141 141 264 264 264 F F F 12 240-259 138 138 138 260 260 260 F F F 13 260-279 144 144 144 272 272 272 F F F 14 280-299 142 142 142 265 265 265 F F F 15 300-319 145 145 145 273 273 273 F F F 16 320-339 145 145 145 273 273 273 F F F 17 340-359 140 140 140 263 263 263 F F F 18 360-379 143 143 143 268 268 268 F F F 19 380-399 143 143 143 268 268 268 F F F 20 400-419 142 142 142 267 267 267 F F F 21 420-439 144 144 144 272 272 272 F F F 22 440-459 136 136 136 257 257 257 F F F 23 460-479 138 138 138 260 260 260 F F F 24 480-499 142 142 142 267 267 267 F F F 25 500-519 144 144 144 270 270 270 F F F 26 520-539 145 145 145 273 273 273 F F F 27 540-559 143 143 143 268 268 268 F F F 28 560-579 145 145 145 273 273 273 F F F 29 580-599 141 141 141 264 264 264 F F F Max gen: 145, Max flush: 273 Begin loop 73 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.07465909Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.119635006Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.120262249Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.12029891Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.12031384Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.120328681Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771} {"msg":"Crucible 5b9bcf29-4118-4da5-9eb1-a3ece79425a0 has session id: 3688b425-f40b-442c-9c72-57176179015d","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.120718119Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"Upstairs opts: Upstairs UUID: 5b9bcf29-4118-4da5-9eb1-a3ece79425a0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.12076338Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"Crucible stats registered with UUID: 5b9bcf29-4118-4da5-9eb1-a3ece79425a0","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.120786121Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.152368406Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.152403216Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771} {"msg":"5b9bcf29-4118-4da5-9eb1-a3ece79425a0 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.152468638Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.152488018Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"client":"0","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.152514069Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"client":"0","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.152529049Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"client":"1","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.152542839Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"client":"1","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.152556Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"client":"2","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.15256957Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"client":"2","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.15258291Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"":"io task","client":"2","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.152596601Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"":"io task","client":"0","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.152619371Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"ds_connection connected from Ok(127.0.0.1:58765)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.152873057Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"ds_connection connected from Ok(127.0.0.1:63200)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.152915468Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"":"io task","client":"2","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"ds_connection connected from Ok(127.0.0.1:49657)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.152948348Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"":"io task","client":"0","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.153601113Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"client":"0","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.153631763Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"client":"0","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.153648583Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"client":"1","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.153663144Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"client":"1","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.153676884Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"client":"2","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.153703385Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"client":"2","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.154186535Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"client":"0","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.154218996Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"client":"1","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.154332898Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"client":"2","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.154761998Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"client":"1","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.154790408Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"client":"1","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.154805799Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.154818789Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"client":"0","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.154832709Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"client":"0","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.154845989Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.15486344Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"client":"2","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.15487691Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"client":"2","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"[0]R flush_numbers[0..12]: [272, 272, 266, 264, 272, 266, 266, 260, 273, 273, 269, 264]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.154907291Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"":"downstairs","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"[0]R generation[0..12]: [144, 144, 142, 141, 144, 142, 142, 138, 145, 145, 144, 141]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.154921981Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"":"downstairs","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.154936111Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"":"downstairs","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"[1]R flush_numbers[0..12]: [272, 272, 266, 264, 272, 266, 266, 260, 273, 273, 269, 264]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.154953372Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"":"downstairs","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"[1]R generation[0..12]: [144, 144, 142, 141, 144, 142, 142, 138, 145, 145, 144, 141]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.154968722Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"":"downstairs","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.154982932Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"":"downstairs","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"[2]R flush_numbers[0..12]: [272, 272, 266, 264, 272, 266, 266, 260, 273, 273, 269, 264]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.154996713Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"":"downstairs","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"[2]R generation[0..12]: [144, 144, 142, 141, 144, 142, 142, 138, 145, 145, 144, 141]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.155010453Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"":"downstairs","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.155024233Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"":"downstairs","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.155037314Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"":"downstairs","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.155050594Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"":"downstairs","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"Next flush: 274","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.155063564Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"":"downstairs","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.155076825Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"":"downstairs","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:55:29.155090215Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.155109915Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"client":"0","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.155123095Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"client":"1","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.155136236Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"client":"2","":"downstairs","session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.155151746Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.155164406Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"5b9bcf29-4118-4da5-9eb1-a3ece79425a0 is now active with session: 3688b425-f40b-442c-9c72-57176179015d","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.155177207Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.155189327Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771,"session_id":"3688b425-f40b-442c-9c72-57176179015d"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:29.155201787Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 574 len 28672 data: 18 17 12 10 10 9 8 03/30 Read block 590 len 36864 04/30 Write block 398 len 12288 data: 8 11 12 05/30 Read block 92 len 8192 06/30 Write block 312 len 8192 data: 8 8 07/30 Write block 161 len 32768 data: 11 11 11 11 8 9 8 10 08/30 Read block 109 len 12288 09/30 Write block 543 len 40960 data: 10 11 16 15 16 16 13 13 13 16 10/30 Write block 168 len 20480 data: 11 11 9 10 11 11/30 Read block 100 len 4096 12/30 Read block 133 len 28672 13/30 Write block 516 len 4096 data: 12 14/30 Read block 178 len 40960 15/30 Read block 215 len 12288 16/30 Read block 413 len 12288 17/30 Flush 18/30 Write block 411 len 24576 data: 12 11 11 9 9 7 19/30 Write block 134 len 28672 data: 11 12 10 9 9 7 8 20/30 Read block 589 len 24576 21/30 Write block 120 len 32768 data: 8 10 8 12 11 10 11 9 22/30 Read block 573 len 20480 23/30 Read block 91 len 24576 24/30 Read block 171 len 4096 25/30 Read block 333 len 36864 26/30 Write block 38 len 28672 data: 14 15 14 11 11 10 10 27/30 Read block 443 len 24576 28/30 Read block 570 len 12288 29/30 Read block 538 len 12288 30/30 Read block 82 len 24576 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 6 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 6 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:55:29.175795954Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1771} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 272 272 272 F F F 1 020-039 144 144 144 272 272 272 T T T 2 040-059 142 142 142 266 266 266 T T T 3 060-079 141 141 141 264 264 264 F F F 4 080-099 144 144 144 272 272 272 F F F 5 100-119 142 142 142 266 266 266 F F F 6 120-139 142 142 142 266 266 266 T T T 7 140-159 138 138 138 260 260 260 T T T 8 160-179 146 146 146 275 275 275 F F F 9 180-199 145 145 145 273 273 273 F F F 10 200-219 144 144 144 269 269 269 F F F 11 220-239 141 141 141 264 264 264 F F F 12 240-259 138 138 138 260 260 260 F F F 13 260-279 144 144 144 272 272 272 F F F 14 280-299 142 142 142 265 265 265 F F F 15 300-319 146 146 146 275 275 275 F F F 16 320-339 145 145 145 273 273 273 F F F 17 340-359 140 140 140 263 263 263 F F F 18 360-379 143 143 143 268 268 268 F F F 19 380-399 146 146 146 275 275 275 F F F 20 400-419 146 146 146 275 275 275 T T T 21 420-439 144 144 144 272 272 272 F F F 22 440-459 136 136 136 257 257 257 F F F 23 460-479 138 138 138 260 260 260 F F F 24 480-499 142 142 142 267 267 267 F F F 25 500-519 146 146 146 275 275 275 F F F 26 520-539 145 145 145 273 273 273 F F F 27 540-559 146 146 146 275 275 275 F F F 28 560-579 146 146 146 275 275 275 F F F 29 580-599 146 146 146 275 275 275 F F F Max gen: 146, Max flush: 275 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 272 272 272 F F F 1 020-039 147 147 147 276 276 276 F F F 2 040-059 147 147 147 276 276 276 F F F 3 060-079 141 141 141 264 264 264 F F F 4 080-099 144 144 144 272 272 272 F F F 5 100-119 142 142 142 266 266 266 F F F 6 120-139 147 147 147 276 276 276 F F F 7 140-159 147 147 147 276 276 276 F F F 8 160-179 146 146 146 275 275 275 F F F 9 180-199 145 145 145 273 273 273 F F F 10 200-219 144 144 144 269 269 269 F F F 11 220-239 141 141 141 264 264 264 F F F 12 240-259 138 138 138 260 260 260 F F F 13 260-279 144 144 144 272 272 272 F F F 14 280-299 142 142 142 265 265 265 F F F 15 300-319 146 146 146 275 275 275 F F F 16 320-339 145 145 145 273 273 273 F F F 17 340-359 140 140 140 263 263 263 F F F 18 360-379 143 143 143 268 268 268 F F F 19 380-399 146 146 146 275 275 275 F F F 20 400-419 147 147 147 276 276 276 F F F 21 420-439 144 144 144 272 272 272 F F F 22 440-459 136 136 136 257 257 257 F F F 23 460-479 138 138 138 260 260 260 F F F 24 480-499 142 142 142 267 267 267 F F F 25 500-519 146 146 146 275 275 275 F F F 26 520-539 145 145 145 273 273 273 F F F 27 540-559 146 146 146 275 275 275 F F F 28 560-579 146 146 146 275 275 275 F F F 29 580-599 146 146 146 275 275 275 F F F Max gen: 147, Max flush: 276 Begin loop 74 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.444512076Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:55:33.487274473Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.487633021Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.519090303Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.519631265Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.519665956Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.519679806Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.519694167Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779} {"msg":"Crucible bba29f49-8d63-440e-9301-0665efadfdd1 has session id: e83bfc93-1738-49e9-8861-54839a0bf38b","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.520093845Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"Upstairs opts: Upstairs UUID: bba29f49-8d63-440e-9301-0665efadfdd1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.520138716Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"Crucible stats registered with UUID: bba29f49-8d63-440e-9301-0665efadfdd1","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.520163557Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.550940824Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.550972685Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779} {"msg":"bba29f49-8d63-440e-9301-0665efadfdd1 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.551041236Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.551063937Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"client":"0","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.551091808Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"client":"0","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.551116818Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"client":"1","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.551150509Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"client":"1","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.55118526Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"client":"2","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.55122034Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"client":"2","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.551253451Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"":"io task","client":"2","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.551444945Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"":"io task","client":"0","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.551485116Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"":"io task","client":"1","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"ds_connection connected from Ok(127.0.0.1:50515)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.551523477Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"":"io task","client":"0","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"ds_connection connected from Ok(127.0.0.1:63322)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.551558038Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"":"io task","client":"2","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"ds_connection connected from Ok(127.0.0.1:34478)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.551582338Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"":"io task","client":"1","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.552041348Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"client":"2","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.552071849Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"client":"2","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.552099859Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"client":"1","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.55212771Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"client":"1","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.552206082Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"client":"0","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.552289783Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"client":"0","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.552698083Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"client":"1","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.552813145Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"client":"2","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.552844876Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"client":"0","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.553200663Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"client":"1","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.553230554Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"client":"1","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.553246884Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.553425448Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"client":"2","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.553453569Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"client":"2","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.553467989Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.553480589Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"client":"0","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.55349432Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"client":"0","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"[0]R flush_numbers[0..12]: [272, 276, 276, 264, 272, 266, 276, 276, 275, 273, 269, 264]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.55350834Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"":"downstairs","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"[0]R generation[0..12]: [144, 147, 147, 141, 144, 142, 147, 147, 146, 145, 144, 141]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.553538571Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"":"downstairs","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.553553491Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"":"downstairs","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"[1]R flush_numbers[0..12]: [272, 276, 276, 264, 272, 266, 276, 276, 275, 273, 269, 264]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.553570491Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"":"downstairs","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"[1]R generation[0..12]: [144, 147, 147, 141, 144, 142, 147, 147, 146, 145, 144, 141]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.553584752Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"":"downstairs","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.553598832Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"":"downstairs","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"[2]R flush_numbers[0..12]: [272, 276, 276, 264, 272, 266, 276, 276, 275, 273, 269, 264]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.553612862Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"":"downstairs","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"[2]R generation[0..12]: [144, 147, 147, 141, 144, 142, 147, 147, 146, 145, 144, 141]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.553626683Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"":"downstairs","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.553640493Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"":"downstairs","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.553653553Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"":"downstairs","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.553666984Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"":"downstairs","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"Next flush: 277","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:55:33.553680104Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"":"downstairs","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.553711144Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"":"downstairs","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.553733365Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.553753125Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"client":"0","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.553778496Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"client":"1","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.553804867Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"client":"2","":"downstairs","session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.553831497Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.553860808Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"bba29f49-8d63-440e-9301-0665efadfdd1 is now active with session: e83bfc93-1738-49e9-8861-54839a0bf38b","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.553883848Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.553897858Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779,"session_id":"e83bfc93-1738-49e9-8861-54839a0bf38b"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:33.553920609Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 511 len 40960 data: 13 14 12 12 13 13 12 11 9 9 02/30 Read block 94 len 24576 03/30 Read block 379 len 28672 04/30 Flush 05/30 Write block 240 len 8192 data: 11 9 06/30 Write block 246 len 16384 data: 16 16 18 16 07/30 Write block 18 len 16384 data: 12 17 15 17 08/30 Write block 214 len 20480 data: 15 14 13 10 12 09/30 Flush 10/30 Flush 11/30 Write block 220 len 32768 data: 9 12 13 13 11 10 8 10 12/30 Write block 453 len 20480 data: 12 16 13 14 10 13/30 Read block 585 len 4096 14/30 Write block 33 len 40960 data: 8 9 11 11 12 15 16 15 12 12 15/30 Write block 157 len 16384 data: 10 10 10 10 16/30 Write block 552 len 40960 data: 17 15 13 14 12 10 10 11 9 6 17/30 Write block 238 len 4096 data: 11 18/30 Flush 19/30 Write block 5 len 20480 data: 7 8 9 12 12 20/30 Write block 165 len 24576 data: 9 10 9 12 12 10 21/30 Read block 451 len 8192 22/30 Read block 430 len 36864 23/30 Read block 363 len 12288 24/30 Write block 221 len 16384 data: 13 14 14 12 25/30 Write block 431 len 12288 data: 13 12 14 26/30 Read block 431 len 16384 27/30 Write block 130 len 20480 data: 12 13 13 11 12 28/30 Write block 215 len 12288 data: 15 14 11 29/30 Write block 151 len 32768 data: 15 15 13 13 14 10 11 11 30/30 Read block 550 len 4096 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 5 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:55:33.57192498Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1779} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 278 278 278 T T T 1 020-039 148 148 148 280 280 280 F F F 2 040-059 148 148 148 280 280 280 F F F 3 060-079 141 141 141 264 264 264 F F F 4 080-099 144 144 144 272 272 272 F F F 5 100-119 142 142 142 266 266 266 F F F 6 120-139 147 147 147 276 276 276 T T T 7 140-159 148 148 148 280 280 280 T T T 8 160-179 148 148 148 280 280 280 T T T 9 180-199 145 145 145 273 273 273 F F F 10 200-219 148 148 148 278 278 278 T T T 11 220-239 148 148 148 280 280 280 T T T 12 240-259 148 148 148 278 278 278 F F F 13 260-279 144 144 144 272 272 272 F F F 14 280-299 142 142 142 265 265 265 F F F 15 300-319 146 146 146 275 275 275 F F F 16 320-339 145 145 145 273 273 273 F F F 17 340-359 140 140 140 263 263 263 F F F 18 360-379 143 143 143 268 268 268 F F F 19 380-399 146 146 146 275 275 275 F F F 20 400-419 147 147 147 276 276 276 F F F 21 420-439 144 144 144 272 272 272 T T T 22 440-459 148 148 148 280 280 280 F F F 23 460-479 138 138 138 260 260 260 F F F 24 480-499 142 142 142 267 267 267 F F F 25 500-519 148 148 148 277 277 277 F F F 26 520-539 148 148 148 277 277 277 F F F 27 540-559 148 148 148 280 280 280 F F F 28 560-579 148 148 148 280 280 280 F F F 29 580-599 146 146 146 275 275 275 F F F Max gen: 148, Max flush: 280 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 281 281 281 F F F 1 020-039 148 148 148 280 280 280 F F F 2 040-059 148 148 148 280 280 280 F F F 3 060-079 141 141 141 264 264 264 F F F 4 080-099 144 144 144 272 272 272 F F F 5 100-119 142 142 142 266 266 266 F F F 6 120-139 149 149 149 281 281 281 F F F 7 140-159 149 149 149 281 281 281 F F F 8 160-179 149 149 149 281 281 281 F F F 9 180-199 145 145 145 273 273 273 F F F 10 200-219 149 149 149 281 281 281 F F F 11 220-239 149 149 149 281 281 281 F F F 12 240-259 148 148 148 278 278 278 F F F 13 260-279 144 144 144 272 272 272 F F F 14 280-299 142 142 142 265 265 265 F F F 15 300-319 146 146 146 275 275 275 F F F 16 320-339 145 145 145 273 273 273 F F F 17 340-359 140 140 140 263 263 263 F F F 18 360-379 143 143 143 268 268 268 F F F 19 380-399 146 146 146 275 275 275 F F F 20 400-419 147 147 147 276 276 276 F F F 21 420-439 149 149 149 281 281 281 F F F 22 440-459 148 148 148 280 280 280 F F F 23 460-479 138 138 138 260 260 260 F F F 24 480-499 142 142 142 267 267 267 F F F 25 500-519 148 148 148 277 277 277 F F F 26 520-539 148 148 148 277 277 277 F F F 27 540-559 148 148 148 280 280 280 F F F 28 560-579 148 148 148 280 280 280 F F F 29 580-599 146 146 146 275 275 275 F F F Max gen: 149, Max flush: 281 Begin loop 75 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.872621904Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.916350373Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.916995917Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.917039818Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.917065688Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.917083549Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787} {"msg":"Crucible 8ce36478-090f-458a-9969-485793d428a1 has session id: ecb46ae9-b16c-4a43-a581-08c3ed2f4980","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.917456767Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"Upstairs opts: Upstairs UUID: 8ce36478-090f-458a-9969-485793d428a1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.917504458Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"Crucible stats registered with UUID: 8ce36478-090f-458a-9969-485793d428a1","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.917542618Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.949030702Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.949063152Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787} {"msg":"8ce36478-090f-458a-9969-485793d428a1 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.949141974Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.949165355Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"client":"0","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.949195085Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"client":"0","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.949217246Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"client":"1","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.949249446Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"client":"1","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.949284987Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"client":"2","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.949310178Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"client":"2","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.949327758Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"":"io task","client":"0","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.949346699Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"":"io task","client":"2","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.949378119Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"ds_connection connected from Ok(127.0.0.1:59886)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.94941243Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"ds_connection connected from Ok(127.0.0.1:34136)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.949447881Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"":"io task","client":"0","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"ds_connection connected from Ok(127.0.0.1:45817)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.949485811Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"":"io task","client":"2","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.950066284Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"client":"0","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.950093515Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"client":"0","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.950108475Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"client":"1","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.950122415Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"client":"1","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.950375171Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"client":"2","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.950399561Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"client":"2","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.950845041Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"client":"1","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.950870181Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"client":"0","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.950960334Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"client":"2","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.951495325Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"client":"1","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.951525216Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"client":"1","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.951541156Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.951554376Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"client":"0","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.951568277Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"client":"0","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.951581737Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.951594807Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"client":"2","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.951608188Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"client":"2","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"[0]R flush_numbers[0..12]: [281, 280, 280, 264, 272, 266, 281, 281, 281, 273, 281, 281]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.951622288Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"":"downstairs","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"[0]R generation[0..12]: [149, 148, 148, 141, 144, 142, 149, 149, 149, 145, 149, 149]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.951636458Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"":"downstairs","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.951650749Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"":"downstairs","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"[1]R flush_numbers[0..12]: [281, 280, 280, 264, 272, 266, 281, 281, 281, 273, 281, 281]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.951664969Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"":"downstairs","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"[1]R generation[0..12]: [149, 148, 148, 141, 144, 142, 149, 149, 149, 145, 149, 149]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.951679619Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"":"downstairs","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.951698279Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"":"downstairs","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"[2]R flush_numbers[0..12]: [281, 280, 280, 264, 272, 266, 281, 281, 281, 273, 281, 281]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.95171281Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"":"downstairs","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"[2]R generation[0..12]: [149, 148, 148, 141, 144, 142, 149, 149, 149, 145, 149, 149]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.95172762Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"":"downstairs","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.951741971Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"":"downstairs","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.951755911Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"":"downstairs","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-11-28T04:55:37.951769471Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"":"downstairs","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"Next flush: 282","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.951791221Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"":"downstairs","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.951804322Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"":"downstairs","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.951817582Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.951829952Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"client":"0","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.951843163Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"client":"1","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.951857203Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"client":"2","":"downstairs","session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.951870593Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.951883114Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"8ce36478-090f-458a-9969-485793d428a1 is now active with session: ecb46ae9-b16c-4a43-a581-08c3ed2f4980","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.951895834Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.951910624Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787,"session_id":"ecb46ae9-b16c-4a43-a581-08c3ed2f4980"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:37.951923535Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 396 len 36864 data: 13 10 9 12 13 14 12 12 13 02/30 Write block 319 len 28672 data: 7 6 7 7 9 8 9 03/30 Read block 144 len 28672 04/30 Read block 543 len 36864 05/30 Write block 326 len 36864 data: 9 11 8 7 9 10 9 9 12 06/30 Read block 144 len 12288 07/30 Read block 553 len 32768 08/30 Read block 114 len 16384 09/30 Read block 472 len 32768 10/30 Read block 441 len 32768 11/30 Write block 160 len 20480 data: 11 12 12 12 12 12/30 Read block 516 len 28672 13/30 Read block 207 len 8192 14/30 Write block 490 len 8192 data: 11 10 15/30 Read block 246 len 40960 16/30 Write block 192 len 12288 data: 13 12 15 17/30 Read block 289 len 4096 18/30 Read block 363 len 32768 19/30 Read block 85 len 4096 20/30 Write block 473 len 40960 data: 8 12 13 14 13 13 12 13 12 13 21/30 Read block 82 len 20480 22/30 Read block 187 len 24576 23/30 Read block 84 len 36864 24/30 Read block 568 len 28672 25/30 Read block 193 len 16384 26/30 Read block 457 len 4096 27/30 Read block 521 len 12288 28/30 Write block 228 len 24576 data: 8 10 10 11 14 16 29/30 Write block 577 len 20480 data: 11 11 10 9 8 30/30 Read block 38 len 16384 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Read 9 Done Done Done false 1004 Acked Write 9 Done Done Done false 1005 Acked Read 3 Done Done Done false 1006 Acked Read 8 Done Done Done false 1007 Acked Read 4 Done Done Done false 1008 Acked Read 8 Done Done Done false 1009 Acked Read 8 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Read 7 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Write 2 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 5 Sent Done Done false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 28 30 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:55:37.972528982Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1787} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 281 281 281 F F F 1 020-039 148 148 148 280 280 280 F F F 2 040-059 148 148 148 280 280 280 F F F 3 060-079 141 141 141 264 264 264 F F F 4 080-099 144 144 144 272 272 272 F F F 5 100-119 142 142 142 266 266 266 F F F 6 120-139 149 149 149 281 281 281 F F F 7 140-159 149 149 149 281 281 281 F F F 8 160-179 149 149 149 281 281 281 T T T 9 180-199 145 145 145 273 273 273 T T T 10 200-219 149 149 149 281 281 281 F F F 11 220-239 149 149 149 281 281 281 T T T 12 240-259 148 148 148 278 278 278 F F F 13 260-279 144 144 144 272 272 272 F F F 14 280-299 142 142 142 265 265 265 F F F 15 300-319 146 146 146 275 275 275 T T T 16 320-339 145 145 145 273 273 273 T T T 17 340-359 140 140 140 263 263 263 F F F 18 360-379 143 143 143 268 268 268 F F F 19 380-399 146 146 146 275 275 275 T T T 20 400-419 147 147 147 276 276 276 T T T 21 420-439 149 149 149 281 281 281 F F F 22 440-459 148 148 148 280 280 280 F F F 23 460-479 138 138 138 260 260 260 T T T 24 480-499 142 142 142 267 267 267 T T T 25 500-519 148 148 148 277 277 277 F F F 26 520-539 148 148 148 277 277 277 F F F 27 540-559 148 148 148 280 280 280 F F F 28 560-579 148 148 148 280 280 280 T T T 29 580-599 146 146 146 275 275 275 T T T Max gen: 149, Max flush: 281 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 281 281 281 F F F 1 020-039 148 148 148 280 280 280 F F F 2 040-059 148 148 148 280 280 280 F F F 3 060-079 141 141 141 264 264 264 F F F 4 080-099 144 144 144 272 272 272 F F F 5 100-119 142 142 142 266 266 266 F F F 6 120-139 149 149 149 281 281 281 F F F 7 140-159 149 149 149 281 281 281 F F F 8 160-179 150 150 150 282 282 282 F F F 9 180-199 150 150 150 282 282 282 F F F 10 200-219 149 149 149 281 281 281 F F F 11 220-239 150 150 150 282 282 282 F F F 12 240-259 148 148 148 278 278 278 F F F 13 260-279 144 144 144 272 272 272 F F F 14 280-299 142 142 142 265 265 265 F F F 15 300-319 150 150 150 282 282 282 F F F 16 320-339 150 150 150 282 282 282 F F F 17 340-359 140 140 140 263 263 263 F F F 18 360-379 143 143 143 268 268 268 F F F 19 380-399 150 150 150 282 282 282 F F F 20 400-419 150 150 150 282 282 282 F F F 21 420-439 149 149 149 281 281 281 F F F 22 440-459 148 148 148 280 280 280 F F F 23 460-479 150 150 150 282 282 282 F F F 24 480-499 150 150 150 282 282 282 F F F 25 500-519 148 148 148 277 277 277 F F F 26 520-539 148 148 148 277 277 277 F F F 27 540-559 148 148 148 280 280 280 F F F 28 560-579 150 150 150 282 282 282 F F F 29 580-599 150 150 150 282 282 282 F F F Max gen: 150, Max flush: 282 Begin loop 76 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.268447423Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.312618721Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.313331736Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.313367297Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.313382668Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.313397528Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795} {"msg":"Crucible a2812700-7152-4e8a-827c-cfdbd6211b47 has session id: 69880343-e84e-4bdc-b7fa-e8ac5ce718fa","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.313788157Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"Upstairs opts: Upstairs UUID: a2812700-7152-4e8a-827c-cfdbd6211b47, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.313835958Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"Crucible stats registered with UUID: a2812700-7152-4e8a-827c-cfdbd6211b47","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.313859688Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.345173327Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.345207578Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795} {"msg":"a2812700-7152-4e8a-827c-cfdbd6211b47 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.345239429Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.345257029Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"client":"0","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.34528472Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"client":"0","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.34530271Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"client":"1","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.345324471Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"client":"1","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.345342071Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"client":"2","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.345361822Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"client":"2","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.345393352Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"":"io task","client":"2","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.345428483Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"":"io task","client":"0","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.345464384Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"":"io task","client":"1","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"ds_connection connected from Ok(127.0.0.1:55945)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.345499614Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"":"io task","client":"0","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"ds_connection connected from Ok(127.0.0.1:44245)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.345550775Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"":"io task","client":"2","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"ds_connection connected from Ok(127.0.0.1:52176)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.345588786Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"":"io task","client":"1","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.346239881Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"client":"2","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.346275281Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"client":"2","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.346305232Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"client":"0","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.346334292Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"client":"0","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.346399584Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"client":"1","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.346433845Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"client":"1","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.346854104Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"client":"0","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.346894445Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"client":"2","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.347001647Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"client":"1","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.347384095Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"client":"2","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.347425156Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"client":"2","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.347455427Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.347521408Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"client":"1","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.347552099Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"client":"1","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.347569109Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.34758327Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"client":"0","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.34759833Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"client":"0","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"[0]R flush_numbers[0..12]: [281, 280, 280, 264, 272, 266, 281, 281, 282, 282, 281, 282]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.34761388Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"":"downstairs","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"[0]R generation[0..12]: [149, 148, 148, 141, 144, 142, 149, 149, 150, 150, 149, 150]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.347629221Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"":"downstairs","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.347644251Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"":"downstairs","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"[1]R flush_numbers[0..12]: [281, 280, 280, 264, 272, 266, 281, 281, 282, 282, 281, 282]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.347659381Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"":"downstairs","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"[1]R generation[0..12]: [149, 148, 148, 141, 144, 142, 149, 149, 150, 150, 149, 150]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.347679132Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"":"downstairs","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.347695282Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"":"downstairs","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"[2]R flush_numbers[0..12]: [281, 280, 280, 264, 272, 266, 281, 281, 282, 282, 281, 282]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.347710312Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"":"downstairs","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"[2]R generation[0..12]: [149, 148, 148, 141, 144, 142, 149, 149, 150, 150, 149, 150]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.347725613Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"":"downstairs","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.347748603Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"":"downstairs","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"Max found gen is 151","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.347763714Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"":"downstairs","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"Generation requested: 152 >= found:151","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.347778464Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"":"downstairs","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"Next flush: 283","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.347792894Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"":"downstairs","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.347807335Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"":"downstairs","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.347821805Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.347836045Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"client":"0","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-28T04:55:42.347850405Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"client":"1","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.347867446Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"client":"2","":"downstairs","session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.347882236Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.347898577Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"a2812700-7152-4e8a-827c-cfdbd6211b47 is now active with session: 69880343-e84e-4bdc-b7fa-e8ac5ce718fa","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.347913507Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.347926937Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795,"session_id":"69880343-e84e-4bdc-b7fa-e8ac5ce718fa"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:42.347939957Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 178 len 36864 data: 9 8 7 6 9 12 15 15 17 02/30 Write block 307 len 28672 data: 9 8 6 6 8 9 9 03/30 Write block 499 len 32768 data: 15 18 16 11 13 14 13 13 04/30 Write block 344 len 8192 data: 10 10 05/30 Read block 500 len 12288 06/30 Read block 530 len 4096 07/30 Write block 132 len 16384 data: 14 12 13 13 08/30 Write block 495 len 24576 data: 11 13 14 16 16 19 09/30 Read block 167 len 20480 10/30 Read block 17 len 32768 11/30 Write block 326 len 12288 data: 10 12 9 12/30 Read block 238 len 24576 13/30 Read block 315 len 16384 14/30 Read block 272 len 8192 15/30 Write block 135 len 8192 data: 14 11 16/30 Read block 431 len 40960 17/30 Read block 397 len 8192 18/30 Read block 457 len 28672 19/30 Write block 247 len 24576 data: 17 19 17 14 13 13 20/30 Write block 565 len 24576 data: 4 6 7 6 10 13 21/30 Read block 135 len 28672 22/30 Read block 522 len 32768 23/30 Read block 451 len 36864 24/30 Read block 172 len 28672 25/30 Read block 136 len 24576 26/30 Write block 539 len 36864 data: 13 10 9 11 11 12 17 16 17 27/30 Read block 206 len 28672 28/30 Read block 38 len 32768 29/30 Write block 18 len 36864 data: 13 18 16 18 15 16 16 14 17 30/30 Write block 439 len 28672 data: 10 13 15 12 13 11 11 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Write 8 Done Done Done false 1003 Acked Write 2 Done Done Done false 1004 Acked Read 3 Done Done Done false 1005 Acked Read 1 Done Done Done false 1006 Acked Write 4 Done Done Done false 1007 Acked Write 6 Done Done Done false 1008 Acked Read 5 Done Done Done false 1009 Acked Read 8 Done Done Done false 1010 Acked Write 3 Done Done Done false 1011 Acked Read 6 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:55:42.367004131Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1795} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 281 281 281 T T T 1 020-039 148 148 148 280 280 280 T T T 2 040-059 148 148 148 280 280 280 F F F 3 060-079 141 141 141 264 264 264 F F F 4 080-099 144 144 144 272 272 272 F F F 5 100-119 142 142 142 266 266 266 F F F 6 120-139 149 149 149 281 281 281 T T T 7 140-159 149 149 149 281 281 281 F F F 8 160-179 150 150 150 282 282 282 T T T 9 180-199 150 150 150 282 282 282 T T T 10 200-219 149 149 149 281 281 281 F F F 11 220-239 150 150 150 282 282 282 F F F 12 240-259 148 148 148 278 278 278 T T T 13 260-279 144 144 144 272 272 272 F F F 14 280-299 142 142 142 265 265 265 F F F 15 300-319 150 150 150 282 282 282 T T T 16 320-339 150 150 150 282 282 282 T T T 17 340-359 140 140 140 263 263 263 T T T 18 360-379 143 143 143 268 268 268 F F F 19 380-399 150 150 150 282 282 282 F F F 20 400-419 150 150 150 282 282 282 F F F 21 420-439 149 149 149 281 281 281 T T T 22 440-459 148 148 148 280 280 280 T T T 23 460-479 150 150 150 282 282 282 F F F 24 480-499 150 150 150 282 282 282 T T T 25 500-519 148 148 148 277 277 277 T T T 26 520-539 148 148 148 277 277 277 T T T 27 540-559 148 148 148 280 280 280 T T T 28 560-579 150 150 150 282 282 282 T T T 29 580-599 150 150 150 282 282 282 F F F Max gen: 150, Max flush: 282 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 283 283 283 F F F 1 020-039 151 151 151 283 283 283 F F F 2 040-059 148 148 148 280 280 280 F F F 3 060-079 141 141 141 264 264 264 F F F 4 080-099 144 144 144 272 272 272 F F F 5 100-119 142 142 142 266 266 266 F F F 6 120-139 151 151 151 283 283 283 F F F 7 140-159 149 149 149 281 281 281 F F F 8 160-179 151 151 151 283 283 283 F F F 9 180-199 151 151 151 283 283 283 F F F 10 200-219 149 149 149 281 281 281 F F F 11 220-239 150 150 150 282 282 282 F F F 12 240-259 151 151 151 283 283 283 F F F 13 260-279 144 144 144 272 272 272 F F F 14 280-299 142 142 142 265 265 265 F F F 15 300-319 151 151 151 283 283 283 F F F 16 320-339 151 151 151 283 283 283 F F F 17 340-359 151 151 151 283 283 283 F F F 18 360-379 143 143 143 268 268 268 F F F 19 380-399 150 150 150 282 282 282 F F F 20 400-419 150 150 150 282 282 282 F F F 21 420-439 151 151 151 283 283 283 F F F 22 440-459 151 151 151 283 283 283 F F F 23 460-479 150 150 150 282 282 282 F F F 24 480-499 151 151 151 283 283 283 F F F 25 500-519 151 151 151 283 283 283 F F F 26 520-539 151 151 151 283 283 283 F F F 27 540-559 151 151 151 283 283 283 F F F 28 560-579 151 151 151 283 283 283 F F F 29 580-599 150 150 150 282 282 282 F F F Max gen: 151, Max flush: 283 Begin loop 77 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.699449525Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:55:46.742385036Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.742742174Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.775088726Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.775619957Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.775656108Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.775671408Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.775687659Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803} {"msg":"Crucible a2a32928-1811-4713-b389-e6e1353d73fa has session id: 744bdd46-0908-494b-a065-b430e53f4ceb","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.776119018Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"Upstairs opts: Upstairs UUID: a2a32928-1811-4713-b389-e6e1353d73fa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.776157309Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"Crucible stats registered with UUID: a2a32928-1811-4713-b389-e6e1353d73fa","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.776175049Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.807191522Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.807223762Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803} {"msg":"a2a32928-1811-4713-b389-e6e1353d73fa active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.807263753Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.807283144Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"client":"0","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.807312804Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"client":"0","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.807331725Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"client":"1","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.807350135Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"client":"1","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.807368156Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"client":"2","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.807386466Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"client":"2","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.807404796Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"":"io task","client":"2","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.807424007Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"":"io task","client":"0","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.807447787Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"":"io task","client":"1","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"ds_connection connected from Ok(127.0.0.1:54095)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.8075687Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"":"io task","client":"0","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"ds_connection connected from Ok(127.0.0.1:51456)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.807664942Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"":"io task","client":"1","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"ds_connection connected from Ok(127.0.0.1:56797)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.807705603Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"":"io task","client":"2","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.808327156Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"client":"1","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.808357607Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"client":"1","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.808376097Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"client":"2","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.808391148Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"client":"2","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.808404778Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"client":"0","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.808423419Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"client":"0","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.80893942Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"client":"1","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.80898302Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"client":"2","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.809139754Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"client":"0","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.809464731Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"client":"2","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.809491882Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"client":"2","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.809506762Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.809594234Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"client":"1","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.809614704Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"client":"1","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.809628574Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.809641915Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"client":"0","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.809655455Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"client":"0","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"[0]R flush_numbers[0..12]: [283, 283, 280, 264, 272, 266, 283, 281, 283, 283, 281, 282]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.809669535Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"":"downstairs","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"[0]R generation[0..12]: [151, 151, 148, 141, 144, 142, 151, 149, 151, 151, 149, 150]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.809687736Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"":"downstairs","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.809702246Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"":"downstairs","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"[1]R flush_numbers[0..12]: [283, 283, 280, 264, 272, 266, 283, 281, 283, 283, 281, 282]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.809715907Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"":"downstairs","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"[1]R generation[0..12]: [151, 151, 148, 141, 144, 142, 151, 149, 151, 151, 149, 150]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.809729537Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"":"downstairs","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.809743527Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"":"downstairs","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"[2]R flush_numbers[0..12]: [283, 283, 280, 264, 272, 266, 283, 281, 283, 283, 281, 282]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.809757357Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"":"downstairs","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"[2]R generation[0..12]: [151, 151, 148, 141, 144, 142, 151, 149, 151, 151, 149, 150]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.809770978Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"":"downstairs","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.809785038Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"":"downstairs","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.809797988Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"":"downstairs","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"Generation requested: 154 >= found:152","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.809811169Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"":"downstairs","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"Next flush: 284","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.809824179Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"":"downstairs","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.809837139Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"":"downstairs","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.809860239Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.80987246Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"client":"0","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.80988783Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"client":"1","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.809901Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"client":"2","":"downstairs","session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.809914281Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.809926691Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"a2a32928-1811-4713-b389-e6e1353d73fa is now active with session: 744bdd46-0908-494b-a065-b430e53f4ceb","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.809939521Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.809951572Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803,"session_id":"744bdd46-0908-494b-a065-b430e53f4ceb"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:46.809964732Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 459 len 40960 data: 12 10 11 11 13 10 8 10 11 12 02/30 Write block 62 len 16384 data: 11 10 11 14 03/30 Write block 346 len 16384 data: 9 11 10 10 04/30 Flush 05/30 Write block 63 len 4096 data: 11 06/30 Write block 333 len 36864 data: 10 13 15 17 15 16 12 12 11 07/30 Read block 379 len 4096 08/30 Write block 315 len 8192 data: 12 8 09/30 Write block 25 len 40960 data: 15 18 16 15 12 8 8 9 9 10 10/30 Read block 334 len 24576 11/30 Write block 215 len 8192 data: 16 15 12/30 Write block 439 len 32768 data: 11 14 16 13 14 12 12 8 13/30 Read block 130 len 8192 14/30 Write block 388 len 4096 data: 11 15/30 Write block 183 len 32768 data: 13 16 16 18 16 14 15 14 16/30 Read block 91 len 32768 17/30 Write block 27 len 4096 data: 17 18/30 Flush 19/30 Write block 247 len 16384 data: 18 20 18 15 20/30 Flush 21/30 Flush 22/30 Flush 23/30 Write block 148 len 40960 data: 14 16 16 16 16 14 14 15 11 12 24/30 Write block 572 len 16384 data: 15 17 19 18 25/30 Write block 413 len 8192 data: 12 10 26/30 Read block 264 len 24576 27/30 Read block 386 len 20480 28/30 Write block 430 len 36864 data: 13 14 13 15 12 13 12 8 9 29/30 Write block 345 len 36864 data: 11 10 12 11 11 10 10 10 6 30/30 Write block 376 len 36864 data: 17 17 17 18 17 14 11 9 8 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 10 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 9 Done Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 2 5 Done 7 6 6 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:55:46.82829961Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1803} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 283 283 283 F F F 1 020-039 154 154 154 285 285 285 F F F 2 040-059 148 148 148 280 280 280 F F F 3 060-079 154 154 154 285 285 285 F F F 4 080-099 144 144 144 272 272 272 F F F 5 100-119 142 142 142 266 266 266 F F F 6 120-139 151 151 151 283 283 283 F F F 7 140-159 149 149 149 281 281 281 T T T 8 160-179 151 151 151 283 283 283 F F F 9 180-199 154 154 154 285 285 285 F F F 10 200-219 154 154 154 285 285 285 F F F 11 220-239 150 150 150 282 282 282 F F F 12 240-259 154 154 154 286 286 286 F F F 13 260-279 144 144 144 272 272 272 F F F 14 280-299 142 142 142 265 265 265 F F F 15 300-319 154 154 154 285 285 285 F F F 16 320-339 154 154 154 285 285 285 F F F 17 340-359 154 154 154 285 285 285 T T T 18 360-379 143 143 143 268 268 268 T T T 19 380-399 154 154 154 285 285 285 T T T 20 400-419 150 150 150 282 282 282 T T T 21 420-439 154 154 154 285 285 285 T T T 22 440-459 154 154 154 285 285 285 F F F 23 460-479 154 154 154 284 284 284 F F F 24 480-499 151 151 151 283 283 283 F F F 25 500-519 151 151 151 283 283 283 F F F 26 520-539 151 151 151 283 283 283 F F F 27 540-559 151 151 151 283 283 283 F F F 28 560-579 151 151 151 283 283 283 T T T 29 580-599 150 150 150 282 282 282 F F F Max gen: 154, Max flush: 286 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 283 283 283 F F F 1 020-039 154 154 154 285 285 285 F F F 2 040-059 148 148 148 280 280 280 F F F 3 060-079 154 154 154 285 285 285 F F F 4 080-099 144 144 144 272 272 272 F F F 5 100-119 142 142 142 266 266 266 F F F 6 120-139 151 151 151 283 283 283 F F F 7 140-159 155 155 155 287 287 287 F F F 8 160-179 151 151 151 283 283 283 F F F 9 180-199 154 154 154 285 285 285 F F F 10 200-219 154 154 154 285 285 285 F F F 11 220-239 150 150 150 282 282 282 F F F 12 240-259 154 154 154 286 286 286 F F F 13 260-279 144 144 144 272 272 272 F F F 14 280-299 142 142 142 265 265 265 F F F 15 300-319 154 154 154 285 285 285 F F F 16 320-339 154 154 154 285 285 285 F F F 17 340-359 155 155 155 287 287 287 F F F 18 360-379 155 155 155 287 287 287 F F F 19 380-399 155 155 155 287 287 287 F F F 20 400-419 155 155 155 287 287 287 F F F 21 420-439 155 155 155 287 287 287 F F F 22 440-459 154 154 154 285 285 285 F F F 23 460-479 154 154 154 284 284 284 F F F 24 480-499 151 151 151 283 283 283 F F F 25 500-519 151 151 151 283 283 283 F F F 26 520-539 151 151 151 283 283 283 F F F 27 540-559 151 151 151 283 283 283 F F F 28 560-579 155 155 155 287 287 287 F F F 29 580-599 150 150 150 282 282 282 F F F Max gen: 155, Max flush: 287 Begin loop 78 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.137296905Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.180938542Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.181559645Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.181595576Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.181609776Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.181624287Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811} {"msg":"Crucible ffca2ec0-4d4c-42de-8819-71b759b0e047 has session id: 563fb1a0-97d9-4dc3-87a9-f4cb316591be","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.182098007Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"Upstairs opts: Upstairs UUID: ffca2ec0-4d4c-42de-8819-71b759b0e047, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.182143428Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"Crucible stats registered with UUID: ffca2ec0-4d4c-42de-8819-71b759b0e047","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.182168358Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.213060929Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.213088619Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811} {"msg":"ffca2ec0-4d4c-42de-8819-71b759b0e047 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.213158541Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.213177371Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"client":"0","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.213201352Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"client":"0","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.213219432Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"client":"1","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.213233302Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"client":"1","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.213246753Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"client":"2","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.213303114Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"client":"2","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.213344745Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"":"io task","client":"2","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.213372945Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"":"io task","client":"0","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.213389426Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"":"io task","client":"1","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"ds_connection connected from Ok(127.0.0.1:55431)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.213413716Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"":"io task","client":"2","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"ds_connection connected from Ok(127.0.0.1:46292)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.213443297Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"":"io task","client":"0","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"ds_connection connected from Ok(127.0.0.1:55029)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.213481808Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"":"io task","client":"1","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.214185303Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"client":"0","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.214215784Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"client":"0","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.214243114Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"client":"2","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.214271645Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"client":"2","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.214411638Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"client":"1","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.214455149Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"client":"1","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.214880258Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"client":"0","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.214911959Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"client":"2","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.214928229Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"client":"1","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.2154437Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"client":"0","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.215472661Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"client":"0","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.215487421Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.215500122Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"client":"1","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.215513822Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"client":"1","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.215527342Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.215539672Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"client":"2","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.215553123Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"client":"2","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"[0]R flush_numbers[0..12]: [283, 285, 280, 285, 272, 266, 283, 287, 283, 285, 285, 282]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.215566943Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"":"downstairs","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"[0]R generation[0..12]: [151, 154, 148, 154, 144, 142, 151, 155, 151, 154, 154, 150]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.215580783Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"":"downstairs","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.215594653Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"":"downstairs","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"[1]R flush_numbers[0..12]: [283, 285, 280, 285, 272, 266, 283, 287, 283, 285, 285, 282]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.215612484Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"":"downstairs","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"[1]R generation[0..12]: [151, 154, 148, 154, 144, 142, 151, 155, 151, 154, 154, 150]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.215626954Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"":"downstairs","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.215641054Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"":"downstairs","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"[2]R flush_numbers[0..12]: [283, 285, 280, 285, 272, 266, 283, 287, 283, 285, 285, 282]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.215655065Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"":"downstairs","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"[2]R generation[0..12]: [151, 154, 148, 154, 144, 142, 151, 155, 151, 154, 154, 150]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.215668815Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"":"downstairs","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-11-28T04:55:51.215682646Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"":"downstairs","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.215703786Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"":"downstairs","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.215717106Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"":"downstairs","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"Next flush: 288","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.215730227Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"":"downstairs","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.215743187Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"":"downstairs","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.215756697Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.215768987Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"client":"0","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.215782378Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"client":"1","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.215795548Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"client":"2","":"downstairs","session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.215808908Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.215822869Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"ffca2ec0-4d4c-42de-8819-71b759b0e047 is now active with session: 563fb1a0-97d9-4dc3-87a9-f4cb316591be","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.215836169Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.215848209Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811,"session_id":"563fb1a0-97d9-4dc3-87a9-f4cb316591be"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:51.215864029Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 55 len 12288 data: 16 16 16 02/30 Read block 534 len 16384 03/30 Flush 04/30 Write block 525 len 40960 data: 17 18 16 16 17 16 13 12 10 10 05/30 Read block 310 len 40960 06/30 Flush 07/30 Write block 157 len 20480 data: 13 12 11 12 13 08/30 Write block 235 len 8192 data: 14 15 09/30 Write block 348 len 32768 data: 12 12 11 11 11 7 6 8 10/30 Read block 157 len 8192 11/30 Read block 363 len 32768 12/30 Write block 0 len 16384 data: 3 5 6 6 13/30 Write block 507 len 28672 data: 15 14 16 14 14 15 13 14/30 Write block 282 len 40960 data: 9 9 12 11 12 10 8 8 9 6 15/30 Write block 511 len 32768 data: 15 16 14 13 14 14 13 12 16/30 Write block 452 len 24576 data: 10 13 17 14 15 11 17/30 Flush 18/30 Flush 19/30 Write block 164 len 12288 data: 13 10 11 20/30 Flush 21/30 Write block 375 len 32768 data: 13 18 18 18 19 18 15 12 22/30 Read block 397 len 4096 23/30 Write block 460 len 12288 data: 11 12 12 24/30 Read block 492 len 12288 25/30 Read block 284 len 40960 26/30 Write block 165 len 36864 data: 11 12 10 13 13 11 11 12 11 27/30 Read block 456 len 28672 28/30 Read block 576 len 4096 29/30 Write block 152 len 36864 data: 17 15 15 16 12 14 13 12 13 30/30 Write block 35 len 36864 data: 12 12 13 16 17 16 13 13 11 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 8 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 9 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:55:51.235795062Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1811} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 290 290 290 F F F 1 020-039 154 154 154 285 285 285 T T T 2 040-059 156 156 156 288 288 288 T T T 3 060-079 154 154 154 285 285 285 F F F 4 080-099 144 144 144 272 272 272 F F F 5 100-119 142 142 142 266 266 266 F F F 6 120-139 151 151 151 283 283 283 F F F 7 140-159 156 156 156 290 290 290 T T T 8 160-179 156 156 156 292 292 292 T T T 9 180-199 154 154 154 285 285 285 F F F 10 200-219 154 154 154 285 285 285 F F F 11 220-239 156 156 156 290 290 290 F F F 12 240-259 154 154 154 286 286 286 F F F 13 260-279 144 144 144 272 272 272 F F F 14 280-299 156 156 156 290 290 290 F F F 15 300-319 154 154 154 285 285 285 F F F 16 320-339 154 154 154 285 285 285 F F F 17 340-359 156 156 156 290 290 290 F F F 18 360-379 155 155 155 287 287 287 T T T 19 380-399 155 155 155 287 287 287 T T T 20 400-419 155 155 155 287 287 287 F F F 21 420-439 155 155 155 287 287 287 F F F 22 440-459 156 156 156 290 290 290 F F F 23 460-479 154 154 154 284 284 284 T T T 24 480-499 151 151 151 283 283 283 F F F 25 500-519 156 156 156 290 290 290 F F F 26 520-539 156 156 156 289 289 289 F F F 27 540-559 151 151 151 283 283 283 F F F 28 560-579 155 155 155 287 287 287 F F F 29 580-599 150 150 150 282 282 282 F F F Max gen: 156, Max flush: 292 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 290 290 290 F F F 1 020-039 157 157 157 293 293 293 F F F 2 040-059 157 157 157 293 293 293 F F F 3 060-079 154 154 154 285 285 285 F F F 4 080-099 144 144 144 272 272 272 F F F 5 100-119 142 142 142 266 266 266 F F F 6 120-139 151 151 151 283 283 283 F F F 7 140-159 157 157 157 293 293 293 F F F 8 160-179 157 157 157 293 293 293 F F F 9 180-199 154 154 154 285 285 285 F F F 10 200-219 154 154 154 285 285 285 F F F 11 220-239 156 156 156 290 290 290 F F F 12 240-259 154 154 154 286 286 286 F F F 13 260-279 144 144 144 272 272 272 F F F 14 280-299 156 156 156 290 290 290 F F F 15 300-319 154 154 154 285 285 285 F F F 16 320-339 154 154 154 285 285 285 F F F 17 340-359 156 156 156 290 290 290 F F F 18 360-379 157 157 157 293 293 293 F F F 19 380-399 157 157 157 293 293 293 F F F 20 400-419 155 155 155 287 287 287 F F F 21 420-439 155 155 155 287 287 287 F F F 22 440-459 156 156 156 290 290 290 F F F 23 460-479 157 157 157 293 293 293 F F F 24 480-499 151 151 151 283 283 283 F F F 25 500-519 156 156 156 290 290 290 F F F 26 520-539 156 156 156 289 289 289 F F F 27 540-559 151 151 151 283 283 283 F F F 28 560-579 155 155 155 287 287 287 F F F 29 580-599 150 150 150 282 282 282 F F F Max gen: 157, Max flush: 293 Begin loop 79 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.513915707Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:55:55.556533402Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.556893289Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.588672109Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.589220501Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.589256422Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.589270872Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.589285382Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819} {"msg":"Crucible 16fc748f-fe1c-49ba-9e6e-d34e1e8131c1 has session id: ce2dcdc9-0ef5-4660-8769-bb7f92fb1144","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.589676781Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"Upstairs opts: Upstairs UUID: 16fc748f-fe1c-49ba-9e6e-d34e1e8131c1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.589721121Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"Crucible stats registered with UUID: 16fc748f-fe1c-49ba-9e6e-d34e1e8131c1","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.589742572Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.620493949Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.62053843Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819} {"msg":"16fc748f-fe1c-49ba-9e6e-d34e1e8131c1 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.620603631Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.620630652Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"client":"0","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.620656463Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"client":"0","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.620679883Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"client":"1","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.620707364Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"client":"1","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.620732544Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"client":"2","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.620753815Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"client":"2","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.620774385Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"":"io task","client":"2","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.620789695Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"":"io task","client":"0","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.620803846Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"":"io task","client":"1","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"ds_connection connected from Ok(127.0.0.1:61801)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.621054071Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"":"io task","client":"2","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"ds_connection connected from Ok(127.0.0.1:50171)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.621092122Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"":"io task","client":"0","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"ds_connection connected from Ok(127.0.0.1:48029)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.621122283Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"":"io task","client":"1","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.621692375Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"client":"1","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.621734276Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"client":"1","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.621766557Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"client":"2","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.621796117Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"client":"2","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.621824208Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"client":"0","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.621850018Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"client":"0","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.62239847Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"client":"1","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.622428011Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"client":"2","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.622444002Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"client":"0","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.622982253Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"client":"1","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.623024014Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"client":"1","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.623039924Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.623052665Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"client":"2","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.623066275Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"client":"2","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.623079865Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.623092445Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"client":"0","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.623106286Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"client":"0","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"[0]R flush_numbers[0..12]: [290, 293, 293, 285, 272, 266, 283, 293, 293, 285, 285, 290]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.623120076Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"":"downstairs","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"[0]R generation[0..12]: [156, 157, 157, 154, 144, 142, 151, 157, 157, 154, 154, 156]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.623134197Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"":"downstairs","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.623148427Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"":"downstairs","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"[1]R flush_numbers[0..12]: [290, 293, 293, 285, 272, 266, 283, 293, 293, 285, 285, 290]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.623162297Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"":"downstairs","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"[1]R generation[0..12]: [156, 157, 157, 154, 144, 142, 151, 157, 157, 154, 154, 156]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.623179147Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"":"downstairs","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.623193718Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"":"downstairs","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"[2]R flush_numbers[0..12]: [290, 293, 293, 285, 272, 266, 283, 293, 293, 285, 285, 290]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.623207758Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"":"downstairs","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"Wait for a query_work_queue command to finish before sending IO } {"msg":"[2]R generation[0..12]: [156, 157, 157, 154, 144, 142, 151, 157, 157, 154, 154, 156]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.62328563Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"":"downstairs","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.62330441Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"":"downstairs","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.623318081Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"":"downstairs","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.623332431Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"":"downstairs","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"Next flush: 294","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.623345881Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"":"downstairs","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.623359041Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"":"downstairs","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.623372222Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.623384402Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"client":"0","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.623397952Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"client":"1","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.623411132Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"client":"2","":"downstairs","session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.623424393Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.623439323Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"16fc748f-fe1c-49ba-9e6e-d34e1e8131c1 is now active with session: ce2dcdc9-0ef5-4660-8769-bb7f92fb1144","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.623452633Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.623464994Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819,"session_id":"ce2dcdc9-0ef5-4660-8769-bb7f92fb1144"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:55.623477344Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 504 len 4096 data: 15 02/30 Flush 03/30 Write block 481 len 12288 data: 13 14 12 04/30 Read block 57 len 24576 05/30 Write block 290 len 40960 data: 10 7 8 7 7 6 7 8 10 11 06/30 Write block 118 len 40960 data: 13 10 9 11 9 13 12 11 12 10 07/30 Read block 140 len 36864 08/30 Flush 09/30 Write block 447 len 40960 data: 8 10 8 7 10 11 14 18 15 16 10/30 Read block 293 len 8192 11/30 Write block 383 len 28672 data: 10 9 7 6 12 12 12 12/30 Write block 274 len 4096 data: 14 13/30 Write block 410 len 24576 data: 16 13 12 13 11 10 14/30 Read block 480 len 24576 15/30 Read block 557 len 4096 16/30 Read block 332 len 20480 17/30 Read block 496 len 36864 18/30 Write block 577 len 20480 data: 12 12 11 10 9 19/30 Write block 209 len 24576 data: 7 12 13 13 12 16 20/30 Write block 459 len 28672 data: 13 12 13 13 14 11 9 21/30 Read block 160 len 32768 22/30 Write block 266 len 24576 data: 12 12 12 16 19 17 23/30 Read block 100 len 40960 24/30 Write block 19 len 20480 data: 19 17 19 16 17 25/30 Read block 501 len 36864 26/30 Flush 27/30 Write block 50 len 16384 data: 14 14 15 17 28/30 Read block 170 len 24576 29/30 Write block 596 len 12288 data: 5 6 4 30/30 Write block 189 len 20480 data: 16 15 14 14 13 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 4 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 3 Done Done Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 3 3 2 8 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:55:55.643340875Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1819} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 296 296 296 F F F 1 020-039 158 158 158 296 296 296 F F F 2 040-059 157 157 157 293 293 293 T T T 3 060-079 154 154 154 285 285 285 F F F 4 080-099 144 144 144 272 272 272 F F F 5 100-119 158 158 158 295 295 295 F F F 6 120-139 158 158 158 295 295 295 F F F 7 140-159 157 157 157 293 293 293 F F F 8 160-179 157 157 157 293 293 293 F F F 9 180-199 154 154 154 285 285 285 T T T 10 200-219 158 158 158 296 296 296 F F F 11 220-239 156 156 156 290 290 290 F F F 12 240-259 154 154 154 286 286 286 F F F 13 260-279 158 158 158 296 296 296 F F F 14 280-299 158 158 158 295 295 295 F F F 15 300-319 154 154 154 285 285 285 F F F 16 320-339 154 154 154 285 285 285 F F F 17 340-359 156 156 156 290 290 290 F F F 18 360-379 157 157 157 293 293 293 F F F 19 380-399 158 158 158 296 296 296 F F F 20 400-419 158 158 158 296 296 296 F F F 21 420-439 155 155 155 287 287 287 F F F 22 440-459 158 158 158 296 296 296 F F F 23 460-479 158 158 158 296 296 296 F F F 24 480-499 158 158 158 295 295 295 F F F 25 500-519 158 158 158 294 294 294 F F F 26 520-539 156 156 156 289 289 289 F F F 27 540-559 151 151 151 283 283 283 F F F 28 560-579 158 158 158 296 296 296 F F F 29 580-599 158 158 158 296 296 296 T T T Max gen: 158, Max flush: 296 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 296 296 296 F F F 1 020-039 158 158 158 296 296 296 F F F 2 040-059 159 159 159 297 297 297 F F F 3 060-079 154 154 154 285 285 285 F F F 4 080-099 144 144 144 272 272 272 F F F 5 100-119 158 158 158 295 295 295 F F F 6 120-139 158 158 158 295 295 295 F F F 7 140-159 157 157 157 293 293 293 F F F 8 160-179 157 157 157 293 293 293 F F F 9 180-199 159 159 159 297 297 297 F F F 10 200-219 158 158 158 296 296 296 F F F 11 220-239 156 156 156 290 290 290 F F F 12 240-259 154 154 154 286 286 286 F F F 13 260-279 158 158 158 296 296 296 F F F 14 280-299 158 158 158 295 295 295 F F F 15 300-319 154 154 154 285 285 285 F F F 16 320-339 154 154 154 285 285 285 F F F 17 340-359 156 156 156 290 290 290 F F F 18 360-379 157 157 157 293 293 293 F F F 19 380-399 158 158 158 296 296 296 F F F 20 400-419 158 158 158 296 296 296 F F F 21 420-439 155 155 155 287 287 287 F F F 22 440-459 158 158 158 296 296 296 F F F 23 460-479 158 158 158 296 296 296 F F F 24 480-499 158 158 158 295 295 295 F F F 25 500-519 158 158 158 294 294 294 F F F 26 520-539 156 156 156 289 289 289 F F F 27 540-559 151 151 151 283 283 283 F F F 28 560-579 158 158 158 296 296 296 F F F 29 580-599 159 159 159 297 297 297 F F F Max gen: 159, Max flush: 297 Begin loop 80 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.900073766Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.94404687Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.944627252Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.944663593Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.944681583Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.944696654Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827} {"msg":"Crucible 984632d7-d8d4-4b06-a521-ef2dff40fbe1 has session id: 324dd8f0-2cc8-44fb-9f10-3930818478c2","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.945164044Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"Upstairs opts: Upstairs UUID: 984632d7-d8d4-4b06-a521-ef2dff40fbe1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.945213065Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"Crucible stats registered with UUID: 984632d7-d8d4-4b06-a521-ef2dff40fbe1","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.945238936Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.976740999Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.976775909Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827} {"msg":"984632d7-d8d4-4b06-a521-ef2dff40fbe1 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.9768072Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.976824871Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"client":"0","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.976854441Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"client":"0","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.976873642Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"client":"1","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.976891282Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"client":"1","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.976908212Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"client":"2","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.976925073Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"client":"2","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.976945483Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"":"io task","client":"2","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.976963364Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"":"io task","client":"0","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.976980314Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"":"io task","client":"1","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:64146)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.977104227Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"":"io task","client":"2","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:39840)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.977146948Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"":"io task","client":"1","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:53076)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.977173498Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"":"io task","client":"0","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.977858423Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"client":"0","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.977900124Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"client":"0","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.977932385Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"client":"2","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.977963715Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"client":"2","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.977995286Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"client":"1","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.978011376Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"client":"1","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.978544178Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"client":"0","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.978587409Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"client":"2","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.97862337Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"client":"1","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.979132191Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"client":"1","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.979161771Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"client":"1","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.979178032Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.979191462Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"client":"0","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.979205372Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"client":"0","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.979218973Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.979231413Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"client":"2","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.979244813Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"client":"2","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"[0]R flush_numbers[0..12]: [296, 296, 297, 285, 272, 295, 295, 293, 293, 297, 296, 290]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.979259734Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"":"downstairs","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"[0]R generation[0..12]: [158, 158, 159, 154, 144, 158, 158, 157, 157, 159, 158, 156]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.979274224Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"":"downstairs","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.979288754Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"":"downstairs","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"[1]R flush_numbers[0..12]: [296, 296, 297, 285, 272, 295, 295, 293, 293, 297, 296, 290]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.979302944Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"":"downstairs","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"[1]R generation[0..12]: [158, 158, 159, 154, 144, 158, 158, 157, 157, 159, 158, 156]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.979316875Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"":"downstairs","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.979331265Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"":"downstairs","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"[2]R flush_numbers[0..12]: [296, 296, 297, 285, 272, 295, 295, 293, 293, 297, 296, 290]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.979345445Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"":"downstairs","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"[2]R generation[0..12]: [158, 158, 159, 154, 144, 158, 158, 157, 157, 159, 158, 156]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.979363176Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"":"downstairs","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.979379436Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"":"downstairs","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.979393036Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"":"downstairs","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.979406657Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"":"downstairs","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"Next flush: 298","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.979419837Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"":"downstairs","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.979433187Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.979456378Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.979468908Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"client":"0","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.979482278Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"client":"1","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.979496439Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"client":"2","":"downstairs","session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.979509929Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.979522509Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"984632d7-d8d4-4b06-a521-ef2dff40fbe1 is now active with session: 324dd8f0-2cc8-44fb-9f10-3930818478c2","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.97953564Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.97954828Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827,"session_id":"324dd8f0-2cc8-44fb-9f10-3930818478c2"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:55:59.97956107Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 130 len 36864 data: 13 14 15 13 14 15 12 10 10 02/30 Read block 11 len 20480 03/30 Read block 201 len 32768 04/30 Read block 140 len 8192 05/30 Flush 06/30 Write block 236 len 4096 data: 16 07/30 Write block 292 len 36864 data: 9 8 8 7 8 9 11 12 11 08/30 Read block 599 len 4096 09/30 Write block 447 len 12288 data: 9 11 9 10/30 Write block 282 len 12288 data: 10 10 13 11/30 Write block 434 len 24576 data: 13 14 13 9 10 12 12/30 Read block 458 len 16384 13/30 Read block 406 len 12288 14/30 Read block 250 len 16384 15/30 Write block 500 len 20480 data: 20 17 12 14 16 16/30 Read block 514 len 8192 17/30 Write block 468 len 24576 data: 13 12 11 10 9 9 18/30 Read block 577 len 28672 19/30 Read block 430 len 32768 20/30 Read block 477 len 32768 21/30 Write block 480 len 40960 data: 14 14 15 13 12 12 11 11 9 8 22/30 Write block 566 len 28672 data: 7 8 7 11 14 15 16 23/30 Write block 393 len 24576 data: 14 12 13 14 11 10 24/30 Write block 152 len 8192 data: 18 16 25/30 Read block 208 len 20480 26/30 Flush 27/30 Write block 159 len 36864 data: 13 14 14 13 13 14 12 13 11 28/30 Write block 433 len 8192 data: 16 14 29/30 Write block 527 len 36864 data: 17 17 18 17 14 13 11 11 7 30/30 Write block 562 len 36864 data: 5 5 4 5 8 9 8 12 15 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 9 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:55:59.998683525Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1827} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 296 296 296 F F F 1 020-039 158 158 158 296 296 296 F F F 2 040-059 159 159 159 297 297 297 F F F 3 060-079 154 154 154 285 285 285 F F F 4 080-099 144 144 144 272 272 272 F F F 5 100-119 158 158 158 295 295 295 F F F 6 120-139 160 160 160 298 298 298 F F F 7 140-159 160 160 160 299 299 299 T T T 8 160-179 157 157 157 293 293 293 T T T 9 180-199 159 159 159 297 297 297 F F F 10 200-219 158 158 158 296 296 296 F F F 11 220-239 160 160 160 299 299 299 F F F 12 240-259 154 154 154 286 286 286 F F F 13 260-279 158 158 158 296 296 296 F F F 14 280-299 160 160 160 299 299 299 F F F 15 300-319 160 160 160 299 299 299 F F F 16 320-339 154 154 154 285 285 285 F F F 17 340-359 156 156 156 290 290 290 F F F 18 360-379 157 157 157 293 293 293 F F F 19 380-399 160 160 160 299 299 299 F F F 20 400-419 158 158 158 296 296 296 F F F 21 420-439 160 160 160 299 299 299 T T T 22 440-459 160 160 160 299 299 299 F F F 23 460-479 160 160 160 299 299 299 F F F 24 480-499 160 160 160 299 299 299 F F F 25 500-519 160 160 160 299 299 299 F F F 26 520-539 156 156 156 289 289 289 T T T 27 540-559 151 151 151 283 283 283 F F F 28 560-579 160 160 160 299 299 299 T T T 29 580-599 159 159 159 297 297 297 F F F Max gen: 160, Max flush: 299 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 296 296 296 F F F 1 020-039 158 158 158 296 296 296 F F F 2 040-059 159 159 159 297 297 297 F F F 3 060-079 154 154 154 285 285 285 F F F 4 080-099 144 144 144 272 272 272 F F F 5 100-119 158 158 158 295 295 295 F F F 6 120-139 160 160 160 298 298 298 F F F 7 140-159 161 161 161 300 300 300 F F F 8 160-179 161 161 161 300 300 300 F F F 9 180-199 159 159 159 297 297 297 F F F 10 200-219 158 158 158 296 296 296 F F F 11 220-239 160 160 160 299 299 299 F F F 12 240-259 154 154 154 286 286 286 F F F 13 260-279 158 158 158 296 296 296 F F F 14 280-299 160 160 160 299 299 299 F F F 15 300-319 160 160 160 299 299 299 F F F 16 320-339 154 154 154 285 285 285 F F F 17 340-359 156 156 156 290 290 290 F F F 18 360-379 157 157 157 293 293 293 F F F 19 380-399 160 160 160 299 299 299 F F F 20 400-419 158 158 158 296 296 296 F F F 21 420-439 161 161 161 300 300 300 F F F 22 440-459 160 160 160 299 299 299 F F F 23 460-479 160 160 160 299 299 299 F F F 24 480-499 160 160 160 299 299 299 F F F 25 500-519 160 160 160 299 299 299 F F F 26 520-539 161 161 161 300 300 300 F F F 27 540-559 151 151 151 283 283 283 F F F 28 560-579 161 161 161 300 300 300 F F F 29 580-599 159 159 159 297 297 297 F F F Max gen: 161, Max flush: 300 Begin loop 81 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.261224152Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:56:04.30396775Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.304326838Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.336019985Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.336563007Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.336596307Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.336610078Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.336628908Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835} {"msg":"Crucible d54f2e82-d25f-4ca3-9349-0a3f8c5e924d has session id: ac70755d-633c-4d3b-8d94-9318702f4d62","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.337078768Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"Upstairs opts: Upstairs UUID: d54f2e82-d25f-4ca3-9349-0a3f8c5e924d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.337111539Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"Crucible stats registered with UUID: d54f2e82-d25f-4ca3-9349-0a3f8c5e924d","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.337126659Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.367897177Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.367922207Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835} {"msg":"d54f2e82-d25f-4ca3-9349-0a3f8c5e924d active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.368000809Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.368024339Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"client":"0","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.36805105Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"client":"0","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.36807805Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"client":"1","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.368105501Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"client":"1","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.368133642Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"client":"2","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.368162802Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"client":"2","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.368192613Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"":"io task","client":"2","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.368222804Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"":"io task","client":"0","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.368253414Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"":"io task","client":"1","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"ds_connection connected from Ok(127.0.0.1:40199)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.368278485Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"":"io task","client":"1","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"ds_connection connected from Ok(127.0.0.1:51223)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.368303695Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"":"io task","client":"2","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"ds_connection connected from Ok(127.0.0.1:50210)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.368420868Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"":"io task","client":"0","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.369074742Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"client":"1","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.369103543Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"client":"1","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.369118563Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"client":"2","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.369132363Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"client":"2","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.369145944Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"client":"0","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.369159614Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"client":"0","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.369715356Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"client":"2","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.369851589Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"client":"1","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.369936331Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"client":"0","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.370284768Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"client":"2","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.370311769Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"client":"2","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.370332439Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.3703755Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"client":"1","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.370394511Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"client":"1","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.370409451Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.370447662Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"client":"0","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.370466102Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"client":"0","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"[0]R flush_numbers[0..12]: [296, 296, 297, 285, 272, 295, 298, 300, 300, 297, 296, 299]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.370481293Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"":"downstairs","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"[0]R generation[0..12]: [158, 158, 159, 154, 144, 158, 160, 161, 161, 159, 158, 160]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.370496733Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"":"downstairs","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.370511793Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"":"downstairs","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"[1]R flush_numbers[0..12]: [296, 296, 297, 285, 272, 295, 298, 300, 300, 297, 296, 299]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.370527563Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"":"downstairs","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"[1]R generation[0..12]: [158, 158, 159, 154, 144, 158, 160, 161, 161, 159, 158, 160]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.370542804Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"":"downstairs","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.370558094Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"":"downstairs","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"[2]R flush_numbers[0..12]: [296, 296, 297, 285, 272, 295, 298, 300, 300, 297, 296, 299]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.370573175Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"":"downstairs","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"[2]R generation[0..12]: [158, 158, 159, 154, 144, 158, 160, 161, 161, 159, 158, 160]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.370588125Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"":"downstairs","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.370605585Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"":"downstairs","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.370620485Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"":"downstairs","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.370635146Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"":"downstairs","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"Next flush: 301","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.370649506Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"":"downstairs","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.370672427Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"":"downstairs","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.370687437Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.370700967Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"client":"0","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.370714948Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"client":"1","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.370729738Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"client":"2","":"downstairs","session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.370744438Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.370757659Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"d54f2e82-d25f-4ca3-9349-0a3f8c5e924d is now active with session: ac70755d-633c-4d3b-8d94-9318702f4d62","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.370771789Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-28T04:56:04.370785149Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835,"session_id":"ac70755d-633c-4d3b-8d94-9318702f4d62"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:04.3707985Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 289 len 24576 data: 9 11 8 10 9 9 02/30 Read block 478 len 40960 03/30 Write block 79 len 16384 data: 11 9 11 9 04/30 Read block 261 len 8192 05/30 Write block 155 len 40960 data: 17 13 15 14 14 15 15 14 14 15 06/30 Write block 461 len 8192 data: 14 14 07/30 Read block 196 len 4096 08/30 Write block 231 len 4096 data: 12 09/30 Read block 71 len 36864 10/30 Write block 301 len 8192 data: 10 10 11/30 Write block 548 len 28672 data: 17 14 14 14 18 16 14 12/30 Read block 338 len 4096 13/30 Read block 121 len 4096 14/30 Write block 524 len 32768 data: 17 18 19 18 18 19 18 15 15/30 Read block 27 len 4096 16/30 Read block 37 len 36864 17/30 Read block 258 len 12288 18/30 Write block 541 len 12288 data: 10 12 12 19/30 Read block 103 len 36864 20/30 Write block 476 len 40960 data: 15 14 14 13 15 15 16 14 13 13 21/30 Write block 475 len 24576 data: 14 16 15 15 14 16 22/30 Read block 538 len 12288 23/30 Read block 571 len 16384 24/30 Read block 425 len 8192 25/30 Read block 224 len 20480 26/30 Read block 273 len 12288 27/30 Read block 295 len 24576 28/30 Read block 210 len 16384 29/30 Read block 576 len 12288 30/30 Write block 476 len 24576 data: 17 16 16 15 17 16 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Write 10 Done Done Done false 1005 Acked Write 2 Done Done Done false 1006 Acked Read 1 Done Done Done false 1007 Acked Write 1 Done Done Done false 1008 Acked Read 9 Done Done Done false 1009 Acked Write 2 Done Done Done false 1010 Acked Write 7 Done Done Done false 1011 Acked Read 1 Done Done Done false 1012 Acked Read 1 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:56:04.390247791Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1835} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 296 296 296 F F F 1 020-039 158 158 158 296 296 296 F F F 2 040-059 159 159 159 297 297 297 F F F 3 060-079 154 154 154 285 285 285 T T T 4 080-099 144 144 144 272 272 272 T T T 5 100-119 158 158 158 295 295 295 F F F 6 120-139 160 160 160 298 298 298 F F F 7 140-159 161 161 161 300 300 300 T T T 8 160-179 161 161 161 300 300 300 T T T 9 180-199 159 159 159 297 297 297 F F F 10 200-219 158 158 158 296 296 296 F F F 11 220-239 160 160 160 299 299 299 T T T 12 240-259 154 154 154 286 286 286 F F F 13 260-279 158 158 158 296 296 296 F F F 14 280-299 160 160 160 299 299 299 T T T 15 300-319 160 160 160 299 299 299 T T T 16 320-339 154 154 154 285 285 285 F F F 17 340-359 156 156 156 290 290 290 F F F 18 360-379 157 157 157 293 293 293 F F F 19 380-399 160 160 160 299 299 299 F F F 20 400-419 158 158 158 296 296 296 F F F 21 420-439 161 161 161 300 300 300 F F F 22 440-459 160 160 160 299 299 299 F F F 23 460-479 160 160 160 299 299 299 T T T 24 480-499 160 160 160 299 299 299 T T T 25 500-519 160 160 160 299 299 299 F F F 26 520-539 161 161 161 300 300 300 T T T 27 540-559 151 151 151 283 283 283 T T T 28 560-579 161 161 161 300 300 300 F F F 29 580-599 159 159 159 297 297 297 F F F Max gen: 161, Max flush: 300 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 296 296 296 F F F 1 020-039 158 158 158 296 296 296 F F F 2 040-059 159 159 159 297 297 297 F F F 3 060-079 162 162 162 301 301 301 F F F 4 080-099 162 162 162 301 301 301 F F F 5 100-119 158 158 158 295 295 295 F F F 6 120-139 160 160 160 298 298 298 F F F 7 140-159 162 162 162 301 301 301 F F F 8 160-179 162 162 162 301 301 301 F F F 9 180-199 159 159 159 297 297 297 F F F 10 200-219 158 158 158 296 296 296 F F F 11 220-239 162 162 162 301 301 301 F F F 12 240-259 154 154 154 286 286 286 F F F 13 260-279 158 158 158 296 296 296 F F F 14 280-299 162 162 162 301 301 301 F F F 15 300-319 162 162 162 301 301 301 F F F 16 320-339 154 154 154 285 285 285 F F F 17 340-359 156 156 156 290 290 290 F F F 18 360-379 157 157 157 293 293 293 F F F 19 380-399 160 160 160 299 299 299 F F F 20 400-419 158 158 158 296 296 296 F F F 21 420-439 161 161 161 300 300 300 F F F 22 440-459 160 160 160 299 299 299 F F F 23 460-479 162 162 162 301 301 301 F F F 24 480-499 162 162 162 301 301 301 F F F 25 500-519 160 160 160 299 299 299 F F F 26 520-539 162 162 162 301 301 301 F F F 27 540-559 162 162 162 301 301 301 F F F 28 560-579 161 161 161 300 300 300 F F F 29 580-599 159 159 159 297 297 297 F F F Max gen: 162, Max flush: 301 Begin loop 82 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.720366205Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.764374719Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.764969082Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.765006253Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.765024853Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.765040724Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843} {"msg":"Crucible cc543469-9b65-4215-9808-8140819f8a15 has session id: a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.765460603Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"Upstairs opts: Upstairs UUID: cc543469-9b65-4215-9808-8140819f8a15, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.765505724Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"Crucible stats registered with UUID: cc543469-9b65-4215-9808-8140819f8a15","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.765546135Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.796253841Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.796289701Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843} {"msg":"cc543469-9b65-4215-9808-8140819f8a15 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.796311882Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.796329742Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"client":"0","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.796359783Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"client":"0","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.796379573Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"client":"1","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.796401504Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"client":"1","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.796502636Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"client":"2","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.796534487Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"client":"2","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.796564387Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"":"io task","client":"2","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.796591858Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"":"io task","client":"0","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.79668831Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"":"io task","client":"1","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"ds_connection connected from Ok(127.0.0.1:46142)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.79672274Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"":"io task","client":"2","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"ds_connection connected from Ok(127.0.0.1:59203)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.796743931Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"":"io task","client":"0","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"ds_connection connected from Ok(127.0.0.1:56561)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.796759291Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"":"io task","client":"1","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.797350824Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"client":"1","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.797379465Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"client":"1","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.797415266Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"client":"2","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.797428876Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"client":"2","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.797442336Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"client":"0","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.797455967Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"client":"0","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.798111301Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"client":"1","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.798143311Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"client":"0","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.798161752Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"client":"2","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.798655862Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"client":"1","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.798696634Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"client":"1","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.798724344Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.798741434Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"client":"2","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.798756685Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"client":"2","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.798771885Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.798785715Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"client":"0","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.798800376Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"client":"0","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"[0]R flush_numbers[0..12]: [296, 296, 297, 301, 301, 295, 298, 301, 301, 297, 296, 301]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.798815506Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"":"downstairs","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"[0]R generation[0..12]: [158, 158, 159, 162, 162, 158, 160, 162, 162, 159, 158, 162]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.798830696Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"":"downstairs","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.798845887Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"":"downstairs","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"[1]R flush_numbers[0..12]: [296, 296, 297, 301, 301, 295, 298, 301, 301, 297, 296, 301]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.798860977Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"":"downstairs","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"Wait for a query_work_queue command to finish before sending IO [1]R generation[0..12]: [158, 158, 159, 162, 162, 158, 160, 162, 162, 159, 158, 162]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.798885647Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"":"downstairs","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.798913338Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"":"downstairs","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"[2]R flush_numbers[0..12]: [296, 296, 297, 301, 301, 295, 298, 301, 301, 297, 296, 301]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.798940039Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"":"downstairs","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"[2]R generation[0..12]: [158, 158, 159, 162, 162, 158, 160, 162, 162, 159, 158, 162]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.798960149Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"":"downstairs","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.79897616Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"":"downstairs","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"Max found gen is 163","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.79899074Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"":"downstairs","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"Generation requested: 164 >= found:163","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.79900471Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"":"downstairs","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"Next flush: 302","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-28T04:56:08.79901921Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"":"downstairs","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.799036511Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"":"downstairs","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.799051901Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.799065842Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"client":"0","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.799080642Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"client":"1","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.799095482Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"client":"2","":"downstairs","session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.799109902Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.799123623Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"cc543469-9b65-4215-9808-8140819f8a15 is now active with session: a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.799137903Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.799151253Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843,"session_id":"a1cb3b8c-b732-4ff6-bc13-5d9d1ae72ef0"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:08.799164724Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 446 len 32768 data: 9 10 12 10 8 11 12 15 02/30 Read block 170 len 4096 03/30 Write block 221 len 40960 data: 14 15 15 13 11 9 11 9 11 11 04/30 Read block 235 len 4096 05/30 Flush 06/30 Write block 180 len 24576 data: 8 7 10 14 17 17 07/30 Write block 307 len 12288 data: 10 9 7 08/30 Read block 53 len 16384 09/30 Read block 522 len 24576 10/30 Read block 580 len 20480 11/30 Write block 266 len 20480 data: 13 13 13 17 20 12/30 Write block 173 len 32768 data: 12 13 13 14 10 10 9 9 13/30 Write block 202 len 40960 data: 15 13 9 11 11 8 9 8 13 14 14/30 Read block 175 len 40960 15/30 Read block 180 len 32768 16/30 Write block 68 len 16384 data: 17 17 15 20 17/30 Read block 170 len 36864 18/30 Write block 146 len 12288 data: 11 14 15 19/30 Read block 459 len 24576 20/30 Write block 497 len 16384 data: 15 17 17 21 21/30 Read block 196 len 12288 22/30 Write block 561 len 12288 data: 7 6 6 23/30 Write block 471 len 8192 data: 11 10 24/30 Read block 53 len 36864 25/30 Read block 158 len 16384 26/30 Read block 186 len 32768 27/30 Write block 154 len 40960 data: 16 18 14 16 15 15 16 16 15 15 28/30 Read block 451 len 4096 29/30 Write block 316 len 32768 data: 9 7 6 8 7 8 8 10 30/30 Read block 526 len 28672 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 6 Done Done Done false 1006 Acked Write 3 Done Done Done false 1007 Acked Read 4 Done Done Done false 1008 Acked Read 6 Done Done Done false 1009 Acked Read 5 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Write 8 Done Done Done false 1012 Acked Write 10 Done Done Done false 1013 Acked Read 10 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 25 24 23 72 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1004 1004 1004 Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:56:08.819509465Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1843} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 296 296 296 F F F 1 020-039 158 158 158 296 296 296 F F F 2 040-059 159 159 159 297 297 297 F F F 3 060-079 162 162 162 301 301 301 T T T 4 080-099 162 162 162 301 301 301 F F F 5 100-119 158 158 158 295 295 295 F F F 6 120-139 160 160 160 298 298 298 F F F 7 140-159 162 162 162 301 301 301 T T T 8 160-179 162 162 162 301 301 301 T T T 9 180-199 159 159 159 297 297 297 T T T 10 200-219 158 158 158 296 296 296 T T T 11 220-239 164 164 164 302 302 302 F F F 12 240-259 154 154 154 286 286 286 F F F 13 260-279 158 158 158 296 296 296 T T T 14 280-299 162 162 162 301 301 301 F F F 15 300-319 162 162 162 301 301 301 T T T 16 320-339 154 154 154 285 285 285 T T T 17 340-359 156 156 156 290 290 290 F F F 18 360-379 157 157 157 293 293 293 F F F 19 380-399 160 160 160 299 299 299 F F F 20 400-419 158 158 158 296 296 296 F F F 21 420-439 161 161 161 300 300 300 F F F 22 440-459 164 164 164 302 302 302 F F F 23 460-479 162 162 162 301 301 301 T T T 24 480-499 162 162 162 301 301 301 T T T 25 500-519 160 160 160 299 299 299 T T T 26 520-539 162 162 162 301 301 301 F F F 27 540-559 162 162 162 301 301 301 F F F 28 560-579 161 161 161 300 300 300 T T T 29 580-599 159 159 159 297 297 297 F F F Max gen: 164, Max flush: 302 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 296 296 296 F F F 1 020-039 158 158 158 296 296 296 F F F 2 040-059 159 159 159 297 297 297 F F F 3 060-079 165 165 165 303 303 303 F F F 4 080-099 162 162 162 301 301 301 F F F 5 100-119 158 158 158 295 295 295 F F F 6 120-139 160 160 160 298 298 298 F F F 7 140-159 165 165 165 303 303 303 F F F 8 160-179 165 165 165 303 303 303 F F F 9 180-199 165 165 165 303 303 303 F F F 10 200-219 165 165 165 303 303 303 F F F 11 220-239 164 164 164 302 302 302 F F F 12 240-259 154 154 154 286 286 286 F F F 13 260-279 165 165 165 303 303 303 F F F 14 280-299 162 162 162 301 301 301 F F F 15 300-319 165 165 165 303 303 303 F F F 16 320-339 165 165 165 303 303 303 F F F 17 340-359 156 156 156 290 290 290 F F F 18 360-379 157 157 157 293 293 293 F F F 19 380-399 160 160 160 299 299 299 F F F 20 400-419 158 158 158 296 296 296 F F F 21 420-439 161 161 161 300 300 300 F F F 22 440-459 164 164 164 302 302 302 F F F 23 460-479 165 165 165 303 303 303 F F F 24 480-499 165 165 165 303 303 303 F F F 25 500-519 165 165 165 303 303 303 F F F 26 520-539 162 162 162 301 301 301 F F F 27 540-559 162 162 162 301 301 301 F F F 28 560-579 165 165 165 303 303 303 F F F 29 580-599 159 159 159 297 297 297 F F F Max gen: 165, Max flush: 303 Begin loop 83 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.123399749Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.167541777Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.168111649Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.16814735Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.168165061Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.168179921Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851} {"msg":"Crucible edd68807-f3f7-4d17-8f26-f1fb37ca8d4b has session id: 035db599-1c18-43ce-b69f-c556a617ed33","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.168673492Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"Upstairs opts: Upstairs UUID: edd68807-f3f7-4d17-8f26-f1fb37ca8d4b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.168719242Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"Crucible stats registered with UUID: edd68807-f3f7-4d17-8f26-f1fb37ca8d4b","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.168744583Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.199621913Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.199666544Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851} {"msg":"edd68807-f3f7-4d17-8f26-f1fb37ca8d4b active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.199730745Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.199758916Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"client":"0","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.199784866Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"client":"0","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.199800287Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"client":"1","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.199820817Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"client":"1","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.199847978Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"client":"2","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.199873788Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"client":"2","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.199894269Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"":"io task","client":"0","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.199921189Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"":"io task","client":"2","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.19994918Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"":"io task","client":"1","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"ds_connection connected from Ok(127.0.0.1:47621)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.199986031Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"":"io task","client":"1","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"ds_connection connected from Ok(127.0.0.1:49153)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.200013721Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"":"io task","client":"2","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"ds_connection connected from Ok(127.0.0.1:52209)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.200039762Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"":"io task","client":"0","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.200788818Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"client":"1","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.200831219Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"client":"1","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.20086329Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"client":"0","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.20089222Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"client":"0","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.200920481Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"client":"2","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.200946361Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"client":"2","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.201480543Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"client":"1","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.201516944Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"client":"0","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.201596776Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"client":"2","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.202060246Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"client":"1","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.202089306Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"client":"1","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.202104397Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.202117197Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"client":"0","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.202130827Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"client":"0","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.202144467Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.202156628Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"client":"2","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.202170168Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"client":"2","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"[0]R flush_numbers[0..12]: [296, 296, 297, 303, 301, 295, 298, 303, 303, 303, 303, 302]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.202184239Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"":"downstairs","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"[0]R generation[0..12]: [158, 158, 159, 165, 162, 158, 160, 165, 165, 165, 165, 164]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.202198119Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"":"downstairs","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.202211979Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"":"downstairs","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"[1]R flush_numbers[0..12]: [296, 296, 297, 303, 301, 295, 298, 303, 303, 303, 303, 302]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.202225819Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"":"downstairs","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"[1]R generation[0..12]: [158, 158, 159, 165, 162, 158, 160, 165, 165, 165, 165, 164]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.20223959Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"":"downstairs","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.20225368Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"":"downstairs","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"[2]R flush_numbers[0..12]: [296, 296, 297, 303, 301, 295, 298, 303, 303, 303, 303, 302]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.20227048Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"":"downstairs","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"[2]R generation[0..12]: [158, 158, 159, 165, 162, 158, 160, 165, 165, 165, 165, 164]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.202284701Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"":"downstairs","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.202298781Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"":"downstairs","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"Wait for a query_work_queue command to finish before sending IO } {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.202389773Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"":"downstairs","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.202417393Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"":"downstairs","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"Next flush: 304","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.202431924Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"":"downstairs","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.202449694Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"":"downstairs","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.202474435Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.202499425Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"client":"0","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.202520576Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"client":"1","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.202534556Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"client":"2","":"downstairs","session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.202557387Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.202582547Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"edd68807-f3f7-4d17-8f26-f1fb37ca8d4b is now active with session: 035db599-1c18-43ce-b69f-c556a617ed33","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.202609508Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.202625158Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851,"session_id":"035db599-1c18-43ce-b69f-c556a617ed33"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:13.202643118Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 405 len 28672 data: 12 12 10 12 14 17 14 02/30 Write block 196 len 12288 data: 14 17 15 03/30 Write block 486 len 32768 data: 12 12 10 9 12 11 10 8 04/30 Read block 334 len 16384 05/30 Read block 267 len 36864 06/30 Read block 586 len 28672 07/30 Read block 32 len 20480 08/30 Read block 570 len 4096 09/30 Flush 10/30 Write block 53 len 36864 data: 18 16 17 17 17 16 14 14 13 11/30 Read block 333 len 4096 12/30 Read block 560 len 28672 13/30 Read block 172 len 40960 14/30 Read block 204 len 16384 15/30 Read block 402 len 36864 16/30 Read block 66 len 32768 17/30 Write block 296 len 40960 data: 9 10 12 13 12 11 11 8 8 9 18/30 Read block 13 len 36864 19/30 Write block 350 len 28672 data: 12 12 12 8 7 9 9 20/30 Read block 595 len 8192 21/30 Write block 434 len 16384 data: 15 15 14 10 22/30 Read block 530 len 8192 23/30 Write block 317 len 16384 data: 8 7 9 8 24/30 Read block 506 len 16384 25/30 Write block 420 len 16384 data: 8 11 13 12 26/30 Write block 20 len 4096 data: 18 27/30 Read block 128 len 28672 28/30 Read block 560 len 8192 29/30 Read block 474 len 8192 30/30 Read block 521 len 24576 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 9 Done Done Done false 1010 Acked Read 1 Done Done Done false 1011 Acked Read 7 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 6 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 20 21 21 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1008 1008 1008 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:56:13.22252267Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1851} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 296 296 296 F F F 1 020-039 158 158 158 296 296 296 T T T 2 040-059 159 159 159 297 297 297 T T T 3 060-079 165 165 165 303 303 303 T T T 4 080-099 162 162 162 301 301 301 F F F 5 100-119 158 158 158 295 295 295 F F F 6 120-139 160 160 160 298 298 298 F F F 7 140-159 165 165 165 303 303 303 F F F 8 160-179 165 165 165 303 303 303 F F F 9 180-199 166 166 166 304 304 304 F F F 10 200-219 165 165 165 303 303 303 F F F 11 220-239 164 164 164 302 302 302 F F F 12 240-259 154 154 154 286 286 286 F F F 13 260-279 165 165 165 303 303 303 F F F 14 280-299 162 162 162 301 301 301 T T T 15 300-319 165 165 165 303 303 303 T T T 16 320-339 165 165 165 303 303 303 T T T 17 340-359 156 156 156 290 290 290 T T T 18 360-379 157 157 157 293 293 293 F F F 19 380-399 160 160 160 299 299 299 F F F 20 400-419 166 166 166 304 304 304 F F F 21 420-439 161 161 161 300 300 300 T T T 22 440-459 164 164 164 302 302 302 F F F 23 460-479 165 165 165 303 303 303 F F F 24 480-499 166 166 166 304 304 304 F F F 25 500-519 165 165 165 303 303 303 F F F 26 520-539 162 162 162 301 301 301 F F F 27 540-559 162 162 162 301 301 301 F F F 28 560-579 165 165 165 303 303 303 F F F 29 580-599 159 159 159 297 297 297 F F F Max gen: 166, Max flush: 304 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 296 296 296 F F F 1 020-039 167 167 167 305 305 305 F F F 2 040-059 167 167 167 305 305 305 F F F 3 060-079 167 167 167 305 305 305 F F F 4 080-099 162 162 162 301 301 301 F F F 5 100-119 158 158 158 295 295 295 F F F 6 120-139 160 160 160 298 298 298 F F F 7 140-159 165 165 165 303 303 303 F F F 8 160-179 165 165 165 303 303 303 F F F 9 180-199 166 166 166 304 304 304 F F F 10 200-219 165 165 165 303 303 303 F F F 11 220-239 164 164 164 302 302 302 F F F 12 240-259 154 154 154 286 286 286 F F F 13 260-279 165 165 165 303 303 303 F F F 14 280-299 167 167 167 305 305 305 F F F 15 300-319 167 167 167 305 305 305 F F F 16 320-339 167 167 167 305 305 305 F F F 17 340-359 167 167 167 305 305 305 F F F 18 360-379 157 157 157 293 293 293 F F F 19 380-399 160 160 160 299 299 299 F F F 20 400-419 166 166 166 304 304 304 F F F 21 420-439 167 167 167 305 305 305 F F F 22 440-459 164 164 164 302 302 302 F F F 23 460-479 165 165 165 303 303 303 F F F 24 480-499 166 166 166 304 304 304 F F F 25 500-519 165 165 165 303 303 303 F F F 26 520-539 162 162 162 301 301 301 F F F 27 540-559 162 162 162 301 301 301 F F F 28 560-579 165 165 165 303 303 303 F F F 29 580-599 159 159 159 297 297 297 F F F Max gen: 167, Max flush: 305 Begin loop 84 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.506980052Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:56:17.549703599Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.550101068Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.581637582Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.582182094Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.582216705Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.582230555Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.582245955Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859} {"msg":"Crucible 457b8b55-05d0-47cc-8098-065162b9efbb has session id: b9e43867-2eaf-4ffa-8b86-c35acab09409","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.582762736Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"Upstairs opts: Upstairs UUID: 457b8b55-05d0-47cc-8098-065162b9efbb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.582802877Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"Crucible stats registered with UUID: 457b8b55-05d0-47cc-8098-065162b9efbb","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.582819878Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.613568695Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.613594375Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859} {"msg":"457b8b55-05d0-47cc-8098-065162b9efbb active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.613646906Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.613675087Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"client":"0","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.613701668Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"client":"0","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.613719148Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"client":"1","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.613733688Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"client":"1","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.613751579Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"client":"2","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.613766759Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"client":"2","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.613780519Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"":"io task","client":"2","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.61379457Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"":"io task","client":"0","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.61380863Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"":"io task","client":"1","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"ds_connection connected from Ok(127.0.0.1:47009)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.613945123Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"":"io task","client":"2","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"ds_connection connected from Ok(127.0.0.1:38271)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.614064555Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"":"io task","client":"0","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"ds_connection connected from Ok(127.0.0.1:54256)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.614104376Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"":"io task","client":"1","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.614614007Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"client":"1","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.614643028Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"client":"1","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.614795601Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"client":"0","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.614824382Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"client":"0","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.614839752Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"client":"2","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.614853403Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"client":"2","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.615260661Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"client":"1","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.615347183Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"client":"2","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.615552938Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"client":"0","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.615797123Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"client":"1","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.615822304Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"client":"1","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.615837044Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.615921216Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"client":"2","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.615941826Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"client":"2","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.615955367Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.616039508Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"client":"0","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.616058259Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"client":"0","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"[0]R flush_numbers[0..12]: [296, 305, 305, 305, 301, 295, 298, 303, 303, 304, 303, 302]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.616072649Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"":"downstairs","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"[0]R generation[0..12]: [158, 167, 167, 167, 162, 158, 160, 165, 165, 166, 165, 164]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.616088079Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"":"downstairs","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.61610247Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"":"downstairs","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"[1]R flush_numbers[0..12]: [296, 305, 305, 305, 301, 295, 298, 303, 303, 304, 303, 302]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.61611664Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"":"downstairs","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"[1]R generation[0..12]: [158, 167, 167, 167, 162, 158, 160, 165, 165, 166, 165, 164]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.61613053Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"":"downstairs","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.616147821Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"":"downstairs","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"[2]R flush_numbers[0..12]: [296, 305, 305, 305, 301, 295, 298, 303, 303, 304, 303, 302]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.616162151Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"":"downstairs","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"[2]R generation[0..12]: [158, 167, 167, 167, 162, 158, 160, 165, 165, 166, 165, 164]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.616176101Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"":"downstairs","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.616190102Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"":"downstairs","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.616203182Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"":"downstairs","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.616216342Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"":"downstairs","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"Next flush: 306","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.616229353Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"":"downstairs","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.616242163Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"":"downstairs","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.616312264Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.616336505Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"client":"0","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.616353035Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"client":"1","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.616368096Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"client":"2","":"downstairs","session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.616383066Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.616397116Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"457b8b55-05d0-47cc-8098-065162b9efbb is now active with session: b9e43867-2eaf-4ffa-8b86-c35acab09409","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.616411457Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.616428297Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 session_id":"b9e43867-2eaf-4ffa-8b86-c35acab09409"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:17.616446777Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 136 len 20480 data: 13 11 11 8 9 02/30 Read block 438 len 40960 03/30 Write block 229 len 16384 data: 12 12 13 15 04/30 Flush 05/30 Write block 483 len 40960 data: 15 14 14 13 13 11 10 13 12 11 06/30 Flush 07/30 Write block 303 len 32768 data: 9 9 10 9 11 10 8 7 08/30 Write block 0 len 8192 data: 4 6 09/30 Read block 525 len 4096 10/30 Write block 125 len 36864 data: 12 13 11 14 11 14 15 16 14 11/30 Read block 538 len 16384 12/30 Read block 533 len 24576 13/30 Write block 366 len 8192 data: 11 12 14/30 Write block 174 len 12288 data: 14 14 15 15/30 Write block 223 len 40960 data: 16 14 12 10 12 10 13 13 14 16 16/30 Read block 162 len 20480 17/30 Write block 14 len 12288 data: 9 12 11 18/30 Read block 560 len 4096 19/30 Write block 29 len 20480 data: 13 9 9 10 10 20/30 Read block 191 len 4096 21/30 Read block 577 len 20480 22/30 Write block 473 len 32768 data: 10 13 15 18 17 17 16 18 23/30 Write block 166 len 16384 data: 14 12 14 14 24/30 Read block 395 len 4096 25/30 Write block 44 len 4096 data: 11 26/30 Write block 316 len 16384 data: 10 9 8 10 27/30 Write block 355 len 16384 data: 10 10 10 12 28/30 Read block 232 len 4096 29/30 Read block 182 len 24576 30/30 Read block 561 len 16384 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 8 Done Done Done false 1007 Acked Write 2 Done Done Done false 1008 Acked Read 1 Done Done Done false 1009 Acked Write 9 Done Done Done false 1010 Acked Read 4 Done Done Done false 1011 Acked Read 6 Done Done Done false 1012 Acked Write 2 Done Done Done false 1013 Acked Write 3 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 4 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 23 24 24 71 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1005 1005 1005 Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:56:17.634763044Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1859} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 296 296 296 T T T 1 020-039 167 167 167 305 305 305 T T T 2 040-059 167 167 167 305 305 305 T T T 3 060-079 167 167 167 305 305 305 F F F 4 080-099 162 162 162 301 301 301 F F F 5 100-119 158 158 158 295 295 295 F F F 6 120-139 168 168 168 306 306 306 T T T 7 140-159 168 168 168 306 306 306 F F F 8 160-179 165 165 165 303 303 303 T T T 9 180-199 166 166 166 304 304 304 F F F 10 200-219 165 165 165 303 303 303 F F F 11 220-239 168 168 168 306 306 306 T T T 12 240-259 154 154 154 286 286 286 F F F 13 260-279 165 165 165 303 303 303 F F F 14 280-299 167 167 167 305 305 305 F F F 15 300-319 167 167 167 305 305 305 T T T 16 320-339 167 167 167 305 305 305 F F F 17 340-359 167 167 167 305 305 305 T T T 18 360-379 157 157 157 293 293 293 T T T 19 380-399 160 160 160 299 299 299 F F F 20 400-419 166 166 166 304 304 304 F F F 21 420-439 167 167 167 305 305 305 F F F 22 440-459 164 164 164 302 302 302 F F F 23 460-479 165 165 165 303 303 303 T T T 24 480-499 168 168 168 307 307 307 T T T 25 500-519 165 165 165 303 303 303 F F F 26 520-539 162 162 162 301 301 301 F F F 27 540-559 162 162 162 301 301 301 F F F 28 560-579 165 165 165 303 303 303 F F F 29 580-599 159 159 159 297 297 297 F F F Max gen: 168, Max flush: 307 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 169 169 169 308 308 308 F F F 1 020-039 169 169 169 308 308 308 F F F 2 040-059 169 169 169 308 308 308 F F F 3 060-079 167 167 167 305 305 305 F F F 4 080-099 162 162 162 301 301 301 F F F 5 100-119 158 158 158 295 295 295 F F F 6 120-139 169 169 169 308 308 308 F F F 7 140-159 168 168 168 306 306 306 F F F 8 160-179 169 169 169 308 308 308 F F F 9 180-199 166 166 166 304 304 304 F F F 10 200-219 165 165 165 303 303 303 F F F 11 220-239 169 169 169 308 308 308 F F F 12 240-259 154 154 154 286 286 286 F F F 13 260-279 165 165 165 303 303 303 F F F 14 280-299 167 167 167 305 305 305 F F F 15 300-319 169 169 169 308 308 308 F F F 16 320-339 167 167 167 305 305 305 F F F 17 340-359 169 169 169 308 308 308 F F F 18 360-379 169 169 169 308 308 308 F F F 19 380-399 160 160 160 299 299 299 F F F 20 400-419 166 166 166 304 304 304 F F F 21 420-439 167 167 167 305 305 305 F F F 22 440-459 164 164 164 302 302 302 F F F 23 460-479 169 169 169 308 308 308 F F F 24 480-499 169 169 169 308 308 308 F F F 25 500-519 165 165 165 303 303 303 F F F 26 520-539 162 162 162 301 301 301 F F F 27 540-559 162 162 162 301 301 301 F F F 28 560-579 165 165 165 303 303 303 F F F 29 580-599 159 159 159 297 297 297 F F F Max gen: 169, Max flush: 308 Begin loop 85 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:21.971302277Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:56:22.013893911Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.014252959Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.04563072Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.046174411Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.046211412Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.046226652Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.046242603Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867} {"msg":"Crucible 0126ad4f-4aa9-469d-bec5-644ec8135e8b has session id: b0c4838e-5d2d-4a29-a678-52c2c6f08680","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.046703523Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"Upstairs opts: Upstairs UUID: 0126ad4f-4aa9-469d-bec5-644ec8135e8b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.046751734Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"Crucible stats registered with UUID: 0126ad4f-4aa9-469d-bec5-644ec8135e8b","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.046780364Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.077683115Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.077728196Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867} {"msg":"0126ad4f-4aa9-469d-bec5-644ec8135e8b active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.077766757Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.077798977Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"client":"0","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.077849088Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"client":"0","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.07792761Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"client":"1","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.077971551Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"client":"1","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.078026782Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"client":"2","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.078078233Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"client":"2","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.078103774Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"":"io task","client":"0","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.078249777Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"":"io task","client":"2","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.078286038Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"":"io task","client":"1","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"ds_connection connected from Ok(127.0.0.1:43078)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.078316549Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"":"io task","client":"0","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"ds_connection connected from Ok(127.0.0.1:42333)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.078342539Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"":"io task","client":"2","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"ds_connection connected from Ok(127.0.0.1:33985)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.078366659Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"":"io task","client":"1","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.078820809Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"client":"2","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.07885886Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"client":"2","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.078913342Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"client":"1","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.078955662Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"client":"1","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.078974453Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"client":"0","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.078989083Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"client":"0","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.079425933Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"client":"1","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.079456143Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"client":"2","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.079604597Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"client":"0","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.079995435Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"client":"1","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.080036876Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"client":"1","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.080064416Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.080079607Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"client":"2","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.080093587Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"client":"2","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.080107997Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.080120298Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"client":"0","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.080133778Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"client":"0","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"[0]R flush_numbers[0..12]: [308, 308, 308, 305, 301, 295, 308, 306, 308, 304, 303, 308]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.080147868Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"":"downstairs","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"[0]R generation[0..12]: [169, 169, 169, 167, 162, 158, 169, 168, 169, 166, 165, 169]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.080161818Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"":"downstairs","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.080175769Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"":"downstairs","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"[1]R flush_numbers[0..12]: [308, 308, 308, 305, 301, 295, 308, 306, 308, 304, 303, 308]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.080193229Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"":"downstairs","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"[1]R generation[0..12]: [169, 169, 169, 167, 162, 158, 169, 168, 169, 166, 165, 169]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.080207419Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"":"downstairs","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.08022142Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"":"downstairs","":"downstairsWait for a query_work_queue command to finish before sending IO ","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"[2]R flush_numbers[0..12]: [308, 308, 308, 305, 301, 295, 308, 306, 308, 304, 303, 308]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.080249311Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"":"downstairs","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"[2]R generation[0..12]: [169, 169, 169, 167, 162, 158, 169, 168, 169, 166, 165, 169]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.080276921Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"":"downstairs","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.080304492Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"":"downstairs","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.080328812Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"":"downstairs","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.080350523Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"":"downstairs","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"Next flush: 309","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.080371423Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"":"downstairs","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.080386783Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"":"downstairs","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.080411154Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.080431694Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"client":"0","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.080446055Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"client":"1","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.080459555Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"client":"2","":"downstairs","session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.080472995Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.080487926Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"0126ad4f-4aa9-469d-bec5-644ec8135e8b is now active with session: b0c4838e-5d2d-4a29-a678-52c2c6f08680","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.080501186Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.080513526Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867,"session_id":"b0c4838e-5d2d-4a29-a678-52c2c6f08680"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:22.080525966Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 304 len 20480 data: 10 11 10 12 11 03/30 Read block 593 len 8192 04/30 Read block 151 len 16384 05/30 Write block 510 len 20480 data: 15 16 17 15 14 06/30 Flush 07/30 Write block 1 len 8192 data: 7 7 08/30 Write block 236 len 32768 data: 17 13 12 8 12 10 10 13 09/30 Write block 191 len 28672 data: 15 15 14 16 18 15 18 10/30 Flush 11/30 Write block 23 len 36864 data: 18 17 16 19 18 16 14 10 10 12/30 Write block 465 len 8192 data: 10 11 13/30 Flush 14/30 Write block 303 len 8192 data: 10 11 15/30 Read block 182 len 16384 16/30 Write block 557 len 32768 data: 11 11 12 10 8 7 7 5 17/30 Read block 435 len 24576 18/30 Read block 173 len 28672 19/30 Write block 477 len 12288 data: 18 18 17 20/30 Read block 544 len 4096 21/30 Flush 22/30 Write block 580 len 32768 data: 11 10 8 12 11 9 9 7 23/30 Flush 24/30 Flush 25/30 Write block 236 len 8192 data: 18 14 26/30 Read block 409 len 36864 27/30 Write block 21 len 20480 data: 20 17 19 18 17 28/30 Write block 575 len 40960 data: 19 13 13 13 12 12 11 9 13 12 29/30 Read block 500 len 20480 30/30 Read block 131 len 40960 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 2 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 6 5 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:56:22.099799084Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1867} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 311 311 311 F F F 1 020-039 170 170 170 312 312 312 T T T 2 040-059 169 169 169 308 308 308 F F F 3 060-079 167 167 167 305 305 305 F F F 4 080-099 162 162 162 301 301 301 F F F 5 100-119 158 158 158 295 295 295 F F F 6 120-139 169 169 169 308 308 308 F F F 7 140-159 168 168 168 306 306 306 F F F 8 160-179 169 169 169 308 308 308 F F F 9 180-199 170 170 170 311 311 311 F F F 10 200-219 165 165 165 303 303 303 F F F 11 220-239 170 170 170 311 311 311 T T T 12 240-259 170 170 170 311 311 311 F F F 13 260-279 165 165 165 303 303 303 F F F 14 280-299 167 167 167 305 305 305 F F F 15 300-319 170 170 170 313 313 313 F F F 16 320-339 167 167 167 305 305 305 F F F 17 340-359 169 169 169 308 308 308 F F F 18 360-379 169 169 169 308 308 308 F F F 19 380-399 160 160 160 299 299 299 F F F 20 400-419 166 166 166 304 304 304 F F F 21 420-439 167 167 167 305 305 305 F F F 22 440-459 164 164 164 302 302 302 F F F 23 460-479 170 170 170 313 313 313 F F F 24 480-499 169 169 169 308 308 308 F F F 25 500-519 170 170 170 310 310 310 F F F 26 520-539 162 162 162 301 301 301 F F F 27 540-559 170 170 170 313 313 313 F F F 28 560-579 170 170 170 313 313 313 T T T 29 580-599 170 170 170 314 314 314 T T T Max gen: 170, Max flush: 314 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 311 311 311 F F F 1 020-039 171 171 171 315 315 315 F F F 2 040-059 169 169 169 308 308 308 F F F 3 060-079 167 167 167 305 305 305 F F F 4 080-099 162 162 162 301 301 301 F F F 5 100-119 158 158 158 295 295 295 F F F 6 120-139 169 169 169 308 308 308 F F F 7 140-159 168 168 168 306 306 306 F F F 8 160-179 169 169 169 308 308 308 F F F 9 180-199 170 170 170 311 311 311 F F F 10 200-219 165 165 165 303 303 303 F F F 11 220-239 171 171 171 315 315 315 F F F 12 240-259 170 170 170 311 311 311 F F F 13 260-279 165 165 165 303 303 303 F F F 14 280-299 167 167 167 305 305 305 F F F 15 300-319 170 170 170 313 313 313 F F F 16 320-339 167 167 167 305 305 305 F F F 17 340-359 169 169 169 308 308 308 F F F 18 360-379 169 169 169 308 308 308 F F F 19 380-399 160 160 160 299 299 299 F F F 20 400-419 166 166 166 304 304 304 F F F 21 420-439 167 167 167 305 305 305 F F F 22 440-459 164 164 164 302 302 302 F F F 23 460-479 170 170 170 313 313 313 F F F 24 480-499 169 169 169 308 308 308 F F F 25 500-519 170 170 170 310 310 310 F F F 26 520-539 162 162 162 301 301 301 F F F 27 540-559 170 170 170 313 313 313 F F F 28 560-579 171 171 171 315 315 315 F F F 29 580-599 171 171 171 315 315 315 F F F Max gen: 171, Max flush: 315 Begin loop 86 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.388486739Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.432174507Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.43281428Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.432849801Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.432868012Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.432883232Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875} {"msg":"Crucible a627b257-6b56-46e2-bbbb-3ee1fc4cd5d4 has session id: 9bbd91be-6780-4768-8422-99aaa599e449","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.433277011Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"Upstairs opts: Upstairs UUID: a627b257-6b56-46e2-bbbb-3ee1fc4cd5d4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.433319331Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"Crucible stats registered with UUID: a627b257-6b56-46e2-bbbb-3ee1fc4cd5d4","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.433337762Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.465012279Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.46504523Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875} {"msg":"a627b257-6b56-46e2-bbbb-3ee1fc4cd5d4 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.465119671Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.465143512Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"client":"0","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.465172922Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"client":"0","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.465192313Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"client":"1","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.465210763Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"client":"1","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.465230174Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"client":"2","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.465254104Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"client":"2","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.465289945Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"":"io task","client":"2","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.465323826Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"":"io task","client":"0","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.465358677Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"":"io task","client":"1","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"ds_connection connected from Ok(127.0.0.1:58092)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.465394897Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"":"io task","client":"2","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"ds_connection connected from Ok(127.0.0.1:41921)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.465426918Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"":"io task","client":"0","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"ds_connection connected from Ok(127.0.0.1:42582)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.465626442Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"":"io task","client":"1","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.465953499Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"client":"0","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.46599537Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"client":"0","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.466127943Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"client":"1","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.466169274Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"client":"1","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.466344088Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"client":"2","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.466384779Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"client":"2","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.466603093Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"client":"0","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.466766877Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"client":"1","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.467101674Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"client":"2","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.467194006Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"client":"0","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.467215477Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"client":"0","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.467229957Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.467291248Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"client":"1","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.467309979Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"client":"1","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.467323639Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.467528274Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"client":"2","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.467546544Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"client":"2","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"[0]R flush_numbers[0..12]: [311, 315, 308, 305, 301, 295, 308, 306, 308, 311, 303, 315]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.467560744Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"":"downstairs","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"[0]R generation[0..12]: [170, 171, 169, 167, 162, 158, 169, 168, 169, 170, 165, 171]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.467574675Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"":"downstairs","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.467588815Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"":"downstairs","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"[1]R flush_numbers[0..12]: [311, 315, 308, 305, 301, 295, 308, 306, 308, 311, 303, 315]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.467602605Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"":"downstairs","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"[1]R generation[0..12]: [170, 171, 169, 167, 162, 158, 169, 168, 169, 170, 165, 171]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.467616506Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"":"downstairs","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.467630496Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"":"downstairs","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"[2]R flush_numbers[0..12]: [311, 315, 308, 305, 301, 295, 308, 306, 308, 311, 303, 315]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.467644486Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"":"downstairs","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"[2]R generation[0..12]: [170, 171, 169, 167, 162, 158, 169, 168, 169, 170, 165, 171]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.467664047Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"":"downstairs","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.467692317Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"":"downstairs","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:56:26.467719818Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"":"downstairs","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.467752849Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"":"downstairs","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"Next flush: 316","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.467774249Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"":"downstairs","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.467799819Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"":"downstairs","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.46782509Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.467844651Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"client":"0","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.467870521Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"client":"1","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.467894581Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"client":"2","":"downstairs","session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 time":"2024-11-28T04:56:26.467919222Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.467946633Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"a627b257-6b56-46e2-bbbb-3ee1fc4cd5d4 is now active with session: 9bbd91be-6780-4768-8422-99aaa599e449","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.467970203Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.467992784Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875,"session_id":"9bbd91be-6780-4768-8422-99aaa599e449"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:26.468016204Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 53 len 28672 data: 19 17 18 18 18 17 15 02/30 Read block 257 len 40960 03/30 Read block 227 len 28672 04/30 Write block 532 len 28672 data: 14 12 12 8 6 9 13 05/30 Write block 137 len 24576 data: 12 12 9 10 11 10 06/30 Write block 31 len 16384 data: 11 11 11 11 07/30 Read block 416 len 20480 08/30 Write block 187 len 36864 data: 17 15 17 16 16 16 15 17 19 09/30 Read block 203 len 4096 10/30 Read block 61 len 20480 11/30 Read block 78 len 12288 12/30 Write block 43 len 4096 data: 12 13/30 Read block 187 len 40960 14/30 Write block 319 len 20480 data: 11 9 9 9 11 15/30 Flush 16/30 Write block 341 len 8192 data: 12 11 17/30 Read block 115 len 36864 18/30 Read block 243 len 4096 19/30 Read block 558 len 20480 20/30 Read block 144 len 8192 21/30 Flush 22/30 Write block 101 len 36864 data: 12 12 12 12 10 9 9 10 12 23/30 Read block 330 len 20480 24/30 Read block 75 len 8192 25/30 Write block 271 len 4096 data: 18 26/30 Read block 455 len 12288 27/30 Read block 25 len 12288 28/30 Read block 305 len 16384 29/30 Write block 13 len 4096 data: 10 30/30 Write block 88 len 28672 data: 7 10 9 12 13 13 12 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 9 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:56:26.487055617Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1875} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 311 311 311 T T T 1 020-039 172 172 172 316 316 316 F F F 2 040-059 172 172 172 316 316 316 F F F 3 060-079 167 167 167 305 305 305 F F F 4 080-099 162 162 162 301 301 301 T T T 5 100-119 158 158 158 295 295 295 T T T 6 120-139 172 172 172 316 316 316 F F F 7 140-159 172 172 172 316 316 316 F F F 8 160-179 169 169 169 308 308 308 F F F 9 180-199 172 172 172 316 316 316 F F F 10 200-219 165 165 165 303 303 303 F F F 11 220-239 171 171 171 315 315 315 F F F 12 240-259 170 170 170 311 311 311 F F F 13 260-279 165 165 165 303 303 303 T T T 14 280-299 167 167 167 305 305 305 F F F 15 300-319 172 172 172 316 316 316 F F F 16 320-339 172 172 172 316 316 316 F F F 17 340-359 172 172 172 317 317 317 F F F 18 360-379 169 169 169 308 308 308 F F F 19 380-399 160 160 160 299 299 299 F F F 20 400-419 166 166 166 304 304 304 F F F 21 420-439 167 167 167 305 305 305 F F F 22 440-459 164 164 164 302 302 302 F F F 23 460-479 170 170 170 313 313 313 F F F 24 480-499 169 169 169 308 308 308 F F F 25 500-519 170 170 170 310 310 310 F F F 26 520-539 172 172 172 316 316 316 F F F 27 540-559 170 170 170 313 313 313 F F F 28 560-579 171 171 171 315 315 315 F F F 29 580-599 171 171 171 315 315 315 F F F Max gen: 172, Max flush: 317 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 173 173 173 318 318 318 F F F 1 020-039 172 172 172 316 316 316 F F F 2 040-059 172 172 172 316 316 316 F F F 3 060-079 167 167 167 305 305 305 F F F 4 080-099 173 173 173 318 318 318 F F F 5 100-119 173 173 173 318 318 318 F F F 6 120-139 172 172 172 316 316 316 F F F 7 140-159 172 172 172 316 316 316 F F F 8 160-179 169 169 169 308 308 308 F F F 9 180-199 172 172 172 316 316 316 F F F 10 200-219 165 165 165 303 303 303 F F F 11 220-239 171 171 171 315 315 315 F F F 12 240-259 170 170 170 311 311 311 F F F 13 260-279 173 173 173 318 318 318 F F F 14 280-299 167 167 167 305 305 305 F F F 15 300-319 172 172 172 316 316 316 F F F 16 320-339 172 172 172 316 316 316 F F F 17 340-359 172 172 172 317 317 317 F F F 18 360-379 169 169 169 308 308 308 F F F 19 380-399 160 160 160 299 299 299 F F F 20 400-419 166 166 166 304 304 304 F F F 21 420-439 167 167 167 305 305 305 F F F 22 440-459 164 164 164 302 302 302 F F F 23 460-479 170 170 170 313 313 313 F F F 24 480-499 169 169 169 308 308 308 F F F 25 500-519 170 170 170 310 310 310 F F F 26 520-539 172 172 172 316 316 316 F F F 27 540-559 170 170 170 313 313 313 F F F 28 560-579 171 171 171 315 315 315 F F F 29 580-599 171 171 171 315 315 315 F F F Max gen: 173, Max flush: 318 Begin loop 87 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.755433682Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:56:30.79780532Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.798164478Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.829714613Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.830272954Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.830306345Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.830324456Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.830340016Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883} {"msg":"Crucible 6f516a04-5abd-46a8-a52a-5d3080bdbb23 has session id: aaf0be90-a4b1-41d0-bb96-9f768f84d0f6","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.830783396Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"Upstairs opts: Upstairs UUID: 6f516a04-5abd-46a8-a52a-5d3080bdbb23, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.830829957Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"Crucible stats registered with UUID: 6f516a04-5abd-46a8-a52a-5d3080bdbb23","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.830855737Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.861345849Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.861378359Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883} {"msg":"6f516a04-5abd-46a8-a52a-5d3080bdbb23 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.8614135Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.861430381Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"client":"0","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.861458711Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"client":"0","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.861476451Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"client":"1","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.861493422Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"client":"1","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.861509932Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"client":"2","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.861541683Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"client":"2","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.861558683Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"":"io task","client":"0","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.861581424Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"":"io task","client":"2","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.861604794Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"":"io task","client":"1","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:53107)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.861715977Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"":"io task","client":"0","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:53458)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.861766928Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"":"io task","client":"2","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:36957)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.861888971Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"":"io task","client":"1","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.862528814Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"client":"2","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.862559135Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"client":"2","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.862664997Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"client":"1","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.862697788Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"client":"1","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.862716048Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"client":"0","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.862737319Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"client":"0","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.863223059Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"client":"2","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.86325909Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"client":"1","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.863338802Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"client":"0","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.863810572Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"client":"1","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.863842223Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"client":"1","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.863859053Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.863961395Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"client":"0","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.863989376Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"client":"0","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.864005436Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.864019647Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"client":"2","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.864033537Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"client":"2","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"[0]R flush_numbers[0..12]: [318, 316, 316, 305, 318, 318, 316, 316, 308, 316, 303, 315]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.864047747Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"":"downstairs","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"[0]R generation[0..12]: [173, 172, 172, 167, 173, 173, 172, 172, 169, 172, 165, 171]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.864061747Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"":"downstairs","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.864075918Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"":"downstairs","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"[1]R flush_numbers[0..12]: [318, 316, 316, 305, 318, 318, 316, 316, 308, 316, 303, 315]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.864089778Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"":"downstairs","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"[1]R generation[0..12]: [173, 172, 172, 167, 173, 173, 172, 172, 169, 172, 165, 171]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.864103619Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"":"downstairs","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.864117719Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"":"downstairs","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"[2]R flush_numbers[0..12]: [318, 316, 316, 305, 318, 318, 316, 316, 308, 316, 303, 315]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.864131659Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"":"downstairs","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"[2]R generation[0..12]: [173, 172, 172, 167, 173, 173, 172, 172, 169, 172, 165, 171]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.864145429Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"":"downstairs","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.86416335Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"":"downstairs","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:56:30.86417742Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"":"downstairs","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.86419549Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"":"downstairs","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"Next flush: 319","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.864208431Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"":"downstairs","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.864221671Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"":"downstairs","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.864235091Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.864247162Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"client":"0","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.864260222Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"client":"1","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.864273302Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"client":"2","":"downstairs","session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.864286563Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.864298833Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"6f516a04-5abd-46a8-a52a-5d3080bdbb23 is now active with session: aaf0be90-a4b1-41d0-bb96-9f768f84d0f6","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.864311743Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.864323873Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883,"session_id":"aaf0be90-a4b1-41d0-bb96-9f768f84d0f6"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:30.864336293Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 342 len 12288 data: 12 9 11 02/30 Flush 03/30 Write block 508 len 24576 data: 15 17 16 17 18 16 04/30 Write block 288 len 8192 data: 9 10 05/30 Write block 136 len 8192 data: 14 13 06/30 Write block 279 len 28672 data: 8 7 6 11 11 14 12 07/30 Write block 164 len 32768 data: 16 13 15 13 15 15 12 12 08/30 Write block 420 len 16384 data: 9 12 14 13 09/30 Read block 267 len 12288 10/30 Read block 410 len 28672 11/30 Write block 442 len 28672 data: 14 15 13 13 10 11 13 12/30 Write block 487 len 28672 data: 14 12 11 14 13 12 9 13/30 Write block 234 len 8192 data: 16 15 14/30 Read block 75 len 4096 15/30 Read block 94 len 36864 16/30 Flush 17/30 Write block 584 len 8192 data: 13 10 18/30 Read block 299 len 12288 19/30 Write block 489 len 28672 data: 12 15 14 13 10 10 12 20/30 Read block 289 len 36864 21/30 Read block 21 len 16384 22/30 Write block 214 len 12288 data: 17 17 16 23/30 Write block 533 len 40960 data: 13 13 9 7 10 14 14 11 11 13 24/30 Read block 108 len 40960 25/30 Write block 573 len 16384 data: 18 20 20 14 26/30 Read block 279 len 32768 27/30 Write block 443 len 28672 data: 16 14 14 11 12 14 11 28/30 Write block 1 len 36864 data: 8 8 7 5 8 9 10 13 13 29/30 Read block 574 len 12288 30/30 Write block 532 len 28672 data: 15 14 14 10 8 11 15 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 2 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:56:30.88307541Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1883} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 173 173 173 318 318 318 T T T 1 020-039 172 172 172 316 316 316 F F F 2 040-059 172 172 172 316 316 316 F F F 3 060-079 167 167 167 305 305 305 F F F 4 080-099 173 173 173 318 318 318 F F F 5 100-119 173 173 173 318 318 318 F F F 6 120-139 174 174 174 320 320 320 F F F 7 140-159 172 172 172 316 316 316 F F F 8 160-179 174 174 174 320 320 320 F F F 9 180-199 172 172 172 316 316 316 F F F 10 200-219 165 165 165 303 303 303 T T T 11 220-239 174 174 174 320 320 320 F F F 12 240-259 170 170 170 311 311 311 F F F 13 260-279 174 174 174 320 320 320 F F F 14 280-299 174 174 174 320 320 320 F F F 15 300-319 172 172 172 316 316 316 F F F 16 320-339 172 172 172 316 316 316 F F F 17 340-359 174 174 174 319 319 319 F F F 18 360-379 169 169 169 308 308 308 F F F 19 380-399 160 160 160 299 299 299 F F F 20 400-419 166 166 166 304 304 304 F F F 21 420-439 174 174 174 320 320 320 F F F 22 440-459 174 174 174 320 320 320 T T T 23 460-479 170 170 170 313 313 313 F F F 24 480-499 174 174 174 320 320 320 T T T 25 500-519 174 174 174 320 320 320 F F F 26 520-539 172 172 172 316 316 316 T T T 27 540-559 170 170 170 313 313 313 T T T 28 560-579 171 171 171 315 315 315 T T T 29 580-599 171 171 171 315 315 315 T T T Max gen: 174, Max flush: 320 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 321 321 321 F F F 1 020-039 172 172 172 316 316 316 F F F 2 040-059 172 172 172 316 316 316 F F F 3 060-079 167 167 167 305 305 305 F F F 4 080-099 173 173 173 318 318 318 F F F 5 100-119 173 173 173 318 318 318 F F F 6 120-139 174 174 174 320 320 320 F F F 7 140-159 172 172 172 316 316 316 F F F 8 160-179 174 174 174 320 320 320 F F F 9 180-199 172 172 172 316 316 316 F F F 10 200-219 175 175 175 321 321 321 F F F 11 220-239 174 174 174 320 320 320 F F F 12 240-259 170 170 170 311 311 311 F F F 13 260-279 174 174 174 320 320 320 F F F 14 280-299 174 174 174 320 320 320 F F F 15 300-319 172 172 172 316 316 316 F F F 16 320-339 172 172 172 316 316 316 F F F 17 340-359 174 174 174 319 319 319 F F F 18 360-379 169 169 169 308 308 308 F F F 19 380-399 160 160 160 299 299 299 F F F 20 400-419 166 166 166 304 304 304 F F F 21 420-439 174 174 174 320 320 320 F F F 22 440-459 175 175 175 321 321 321 F F F 23 460-479 170 170 170 313 313 313 F F F 24 480-499 175 175 175 321 321 321 F F F 25 500-519 174 174 174 320 320 320 F F F 26 520-539 175 175 175 321 321 321 F F F 27 540-559 175 175 175 321 321 321 F F F 28 560-579 175 175 175 321 321 321 F F F 29 580-599 175 175 175 321 321 321 F F F Max gen: 175, Max flush: 321 Begin loop 88 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.195627602Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:56:35.238173085Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.238530923Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.270097808Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.27068457Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.270717891Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.270736361Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.270751172Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891} {"msg":"Crucible 23cde414-96b9-4099-876f-93a8ebaffd0a has session id: 732227c0-c907-4dbb-9130-5b78c2c8d90b","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.271210422Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"Upstairs opts: Upstairs UUID: 23cde414-96b9-4099-876f-93a8ebaffd0a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.271258453Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"Crucible stats registered with UUID: 23cde414-96b9-4099-876f-93a8ebaffd0a","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.271283833Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.301734414Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.301794965Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891} {"msg":"23cde414-96b9-4099-876f-93a8ebaffd0a active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.301811976Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.301826746Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"client":"0","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.301848867Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"client":"0","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.301865047Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"client":"1","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.301880507Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"client":"1","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.301895708Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"client":"2","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.301910898Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"client":"2","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.301930678Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"":"io task","client":"2","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.301963369Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"":"io task","client":"0","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.30199273Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"":"io task","client":"1","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"ds_connection connected from Ok(127.0.0.1:42660)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.302247885Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"":"io task","client":"1","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"ds_connection connected from Ok(127.0.0.1:36535)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.302293156Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"":"io task","client":"0","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"ds_connection connected from Ok(127.0.0.1:62182)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.302324367Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"":"io task","client":"2","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.302834428Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"client":"2","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.302863758Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"client":"2","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.30295008Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"client":"1","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.302970151Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"client":"1","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.302989561Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"client":"0","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.303005001Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"client":"0","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.303467611Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"client":"1","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.303500262Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"client":"2","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.303758268Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"client":"0","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.304046924Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"client":"1","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.304119956Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"client":"1","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.304138266Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.304151277Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"client":"2","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.304164967Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"client":"2","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.304178227Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.304192297Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"client":"0","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.304205907Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"client":"0","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"[0]R flush_numbers[0..12]: [321, 316, 316, 305, 318, 318, 320, 316, 320, 316, 321, 320]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.304234218Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"":"downstairs","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"[0]R generation[0..12]: [175, 172, 172, 167, 173, 173, 174, 172, 174, 172, 175, 174]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.304248959Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"":"downstairs","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.304263139Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"":"downstairs","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"[1]R flush_numbers[0..12]: [321, 316, 316, 305, 318, 318, 320, 316, 320, 316, 321, 320]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.304277289Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"":"downstairs","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"[1]R generation[0..12]: [175, 172, 172, 167, 173, 173, 174, 172, 174, 172, 175, 174]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.304291169Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"":"downstairs","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.3043054Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"":"downstairs","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"[2]R flush_numbers[0..12]: [321, 316, 316, 305, 318, 318, 320, 316, 320, 316, 321, 320]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.30431946Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"":"downstairs","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"[2]R generation[0..12]: [175, 172, 172, 167, 173, 173, 174, 172, 174, 172, 175, 174]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.30433533Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"":"downstairs","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.304349561Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"":"downstairs","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.304363031Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"":"downstairs","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.304376441Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"":"downstairs","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"Next flush: 322","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.304389691Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"":"downstairs","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.304402712Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"":"downstairs","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2024-11-28T04:56:35.304424032Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.304436403Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"client":"0","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.304449763Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"client":"1","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.304462983Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"client":"2","":"downstairs","session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.304476293Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.304489514Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"23cde414-96b9-4099-876f-93a8ebaffd0a is now active with session: 732227c0-c907-4dbb-9130-5b78c2c8d90b","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.304502434Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.304514514Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891,"session_id":"732227c0-c907-4dbb-9130-5b78c2c8d90b"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:35.304526974Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 458 len 40960 data: 10 14 13 15 15 15 12 11 12 12 03/30 Read block 425 len 24576 04/30 Read block 230 len 40960 05/30 Write block 53 len 16384 data: 20 18 19 19 06/30 Write block 234 len 32768 data: 17 16 19 15 13 9 13 11 07/30 Read block 120 len 40960 08/30 Write block 535 len 20480 data: 11 9 12 16 15 09/30 Read block 177 len 36864 10/30 Read block 407 len 16384 11/30 Flush 12/30 Write block 432 len 4096 data: 14 13/30 Write block 110 len 24576 data: 14 14 16 16 15 15 14/30 Write block 455 len 40960 data: 16 17 12 11 15 14 16 16 16 13 15/30 Read block 9 len 36864 16/30 Read block 215 len 20480 17/30 Read block 350 len 36864 18/30 Flush 19/30 Write block 459 len 24576 data: 16 15 17 17 17 14 20/30 Read block 543 len 8192 21/30 Read block 1 len 24576 22/30 Write block 317 len 16384 data: 10 9 12 10 23/30 Read block 105 len 36864 24/30 Write block 11 len 24576 data: 13 9 11 10 13 12 25/30 Flush 26/30 Write block 472 len 16384 data: 11 11 14 16 27/30 Write block 51 len 24576 data: 15 16 21 19 20 20 28/30 Read block 191 len 12288 29/30 Write block 325 len 36864 data: 10 11 13 10 8 10 11 10 11 30/30 Read block 461 len 20480 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 4 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 9 Sent Done Sent false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 2 4 Done 3 5 3 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:56:35.325732614Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1891} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 325 325 325 F F F 1 020-039 172 172 172 316 316 316 F F F 2 040-059 176 176 176 323 323 323 T T T 3 060-079 167 167 167 305 305 305 F F F 4 080-099 173 173 173 318 318 318 F F F 5 100-119 176 176 176 324 324 324 F F F 6 120-139 174 174 174 320 320 320 F F F 7 140-159 172 172 172 316 316 316 F F F 8 160-179 174 174 174 320 320 320 F F F 9 180-199 172 172 172 316 316 316 F F F 10 200-219 175 175 175 321 321 321 F F F 11 220-239 176 176 176 323 323 323 F F F 12 240-259 176 176 176 323 323 323 F F F 13 260-279 174 174 174 320 320 320 F F F 14 280-299 174 174 174 320 320 320 F F F 15 300-319 176 176 176 325 325 325 F F F 16 320-339 176 176 176 325 325 325 T T T 17 340-359 174 174 174 319 319 319 F F F 18 360-379 169 169 169 308 308 308 F F F 19 380-399 160 160 160 299 299 299 F F F 20 400-419 166 166 166 304 304 304 F F F 21 420-439 176 176 176 324 324 324 F F F 22 440-459 176 176 176 325 325 325 F F F 23 460-479 176 176 176 325 325 325 T T T 24 480-499 175 175 175 321 321 321 F F F 25 500-519 174 174 174 320 320 320 F F F 26 520-539 176 176 176 323 323 323 F F F 27 540-559 175 175 175 321 321 321 F F F 28 560-579 175 175 175 321 321 321 F F F 29 580-599 175 175 175 321 321 321 F F F Max gen: 176, Max flush: 325 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 325 325 325 F F F 1 020-039 172 172 172 316 316 316 F F F 2 040-059 177 177 177 326 326 326 F F F 3 060-079 167 167 167 305 305 305 F F F 4 080-099 173 173 173 318 318 318 F F F 5 100-119 176 176 176 324 324 324 F F F 6 120-139 174 174 174 320 320 320 F F F 7 140-159 172 172 172 316 316 316 F F F 8 160-179 174 174 174 320 320 320 F F F 9 180-199 172 172 172 316 316 316 F F F 10 200-219 175 175 175 321 321 321 F F F 11 220-239 176 176 176 323 323 323 F F F 12 240-259 176 176 176 323 323 323 F F F 13 260-279 174 174 174 320 320 320 F F F 14 280-299 174 174 174 320 320 320 F F F 15 300-319 176 176 176 325 325 325 F F F 16 320-339 177 177 177 326 326 326 F F F 17 340-359 174 174 174 319 319 319 F F F 18 360-379 169 169 169 308 308 308 F F F 19 380-399 160 160 160 299 299 299 F F F 20 400-419 166 166 166 304 304 304 F F F 21 420-439 176 176 176 324 324 324 F F F 22 440-459 176 176 176 325 325 325 F F F 23 460-479 177 177 177 326 326 326 F F F 24 480-499 175 175 175 321 321 321 F F F 25 500-519 174 174 174 320 320 320 F F F 26 520-539 176 176 176 323 323 323 F F F 27 540-559 175 175 175 321 321 321 F F F 28 560-579 175 175 175 321 321 321 F F F 29 580-599 175 175 175 321 321 321 F F F Max gen: 177, Max flush: 326 Begin loop 89 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.613111231Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:56:39.656486912Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.65685323Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.689251803Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.689806445Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.689842665Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.689857116Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.689871766Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899} {"msg":"Crucible f6ca20e2-c2af-4f35-b523-67146599dc44 has session id: f4621a7e-547a-4a9c-8cbf-7da34ab4149e","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.690326846Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"Upstairs opts: Upstairs UUID: f6ca20e2-c2af-4f35-b523-67146599dc44, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.690371577Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"Crucible stats registered with UUID: f6ca20e2-c2af-4f35-b523-67146599dc44","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.690397248Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.721936212Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.721968742Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899} {"msg":"f6ca20e2-c2af-4f35-b523-67146599dc44 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.722018223Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.722039594Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"client":"0","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.722069175Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"client":"0","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.722087045Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"client":"1","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.722111386Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"client":"1","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.722143346Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"client":"2","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.722177177Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"client":"2","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.722210308Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"":"io task","client":"2","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.722243828Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"":"io task","client":"0","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.722275749Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"":"io task","client":"1","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"ds_connection connected from Ok(127.0.0.1:54645)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.7223125Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"":"io task","client":"2","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"ds_connection connected from Ok(127.0.0.1:49614)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.722345771Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"":"io task","client":"1","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"ds_connection connected from Ok(127.0.0.1:41856)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.722545475Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"":"io task","client":"0","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.722969354Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"client":"1","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.723006355Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"client":"1","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.723026956Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"client":"2","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.723110657Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"client":"2","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.723139278Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"client":"0","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.723155908Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"client":"0","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.723644649Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"client":"1","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.72367407Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"client":"0","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.72368976Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"client":"2","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.724208981Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"client":"1","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.724233952Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"client":"1","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.724248432Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.724264102Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"client":"2","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.724278023Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"client":"2","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.724291443Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.724304763Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"client":"0","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.724318064Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"client":"0","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"[0]R flush_numbers[0..12]: [325, 316, 326, 305, 318, 324, 320, 316, 320, 316, 321, 323]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.724331964Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"":"downstairs","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"[0]R generation[0..12]: [176, 172, 177, 167, 173, 176, 174, 172, 174, 172, 175, 176]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.724346034Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"":"downstairs","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.724360124Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"":"downstairs","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"[1]R flush_numbers[0..12]: [325, 316, 326, 305, 318, 324, 320, 316, 320, 316, 321, 323]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.724374145Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"":"downstairs","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"[1]R generation[0..12]: [176, 172, 177, 167, 173, 176, 174, 172, 174, 172, 175, 176]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.724388045Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"":"downstairs","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.724401985Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"":"downstairs","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"[2]R flush_numbers[0..12]: [325, 316, 326, 305, 318, 324, 320, 316, 320, 316, 321, 323]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.724415786Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"":"downstairs","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"[2]R generation[0..12]: [176, 172, 177, 167, 173, 176, 174, 172, 174, 172, 175, 176]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.724436456Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"":"downstairs","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.724463937Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"":"downstairs","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {Wait for a query_work_queue command to finish before sending IO "msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.724503088Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"":"downstairs","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.724529848Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"":"downstairs","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"Next flush: 327","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.724544238Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"":"downstairs","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.724563169Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"":"downstairs","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.724588339Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.72461363Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"client":"0","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.72463417Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"client":"1","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.724647891Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"client":"2","":"downstairs","session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.724671541Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.724696662Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"f6ca20e2-c2af-4f35-b523-67146599dc44 is now active with session: f4621a7e-547a-4a9c-8cbf-7da34ab4149e","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.724723162Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.724738302Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899,"session_id":"f4621a7e-547a-4a9c-8cbf-7da34ab4149e"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:39.724757663Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 331 len 36864 data: 12 11 12 14 16 18 16 17 13 02/30 Flush 03/30 Write block 454 len 12288 data: 19 17 18 04/30 Write block 555 len 24576 data: 15 13 12 12 13 11 05/30 Read block 274 len 32768 06/30 Write block 532 len 40960 data: 16 15 15 12 10 13 17 16 12 12 07/30 Write block 444 len 8192 data: 15 15 08/30 Write block 9 len 32768 data: 14 11 14 10 12 11 14 13 09/30 Read block 401 len 20480 10/30 Read block 103 len 28672 11/30 Write block 306 len 40960 data: 11 13 12 9 8 9 10 10 12 13 12/30 Read block 307 len 40960 13/30 Read block 317 len 32768 14/30 Write block 202 len 24576 data: 16 14 10 12 12 9 15/30 Read block 265 len 24576 16/30 Write block 570 len 40960 data: 16 16 17 19 21 21 15 14 14 13 17/30 Read block 241 len 8192 18/30 Write block 335 len 28672 data: 17 19 17 18 14 13 13 19/30 Write block 168 len 8192 data: 16 16 20/30 Write block 538 len 16384 data: 18 17 13 13 21/30 Read block 198 len 8192 22/30 Write block 194 len 4096 data: 18 23/30 Read block 323 len 12288 24/30 Flush 25/30 Write block 530 len 8192 data: 19 16 26/30 Read block 208 len 24576 27/30 Write block 293 len 32768 data: 10 10 8 10 11 13 14 13 28/30 Read block 44 len 36864 29/30 Read block 426 len 8192 30/30 Read block 100 len 4096 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 2 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:56:39.74487843Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1899} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 328 328 328 F F F 1 020-039 172 172 172 316 316 316 F F F 2 040-059 177 177 177 326 326 326 F F F 3 060-079 167 167 167 305 305 305 F F F 4 080-099 173 173 173 318 318 318 F F F 5 100-119 176 176 176 324 324 324 F F F 6 120-139 174 174 174 320 320 320 F F F 7 140-159 172 172 172 316 316 316 F F F 8 160-179 178 178 178 328 328 328 F F F 9 180-199 178 178 178 328 328 328 F F F 10 200-219 178 178 178 328 328 328 F F F 11 220-239 176 176 176 323 323 323 F F F 12 240-259 176 176 176 323 323 323 F F F 13 260-279 174 174 174 320 320 320 F F F 14 280-299 174 174 174 320 320 320 T T T 15 300-319 178 178 178 328 328 328 T T T 16 320-339 178 178 178 328 328 328 F F F 17 340-359 178 178 178 328 328 328 F F F 18 360-379 169 169 169 308 308 308 F F F 19 380-399 160 160 160 299 299 299 F F F 20 400-419 166 166 166 304 304 304 F F F 21 420-439 176 176 176 324 324 324 F F F 22 440-459 178 178 178 328 328 328 F F F 23 460-479 177 177 177 326 326 326 F F F 24 480-499 175 175 175 321 321 321 F F F 25 500-519 174 174 174 320 320 320 F F F 26 520-539 178 178 178 328 328 328 T T T 27 540-559 178 178 178 328 328 328 F F F 28 560-579 178 178 178 328 328 328 F F F 29 580-599 175 175 175 321 321 321 F F F Max gen: 178, Max flush: 328 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 328 328 328 F F F 1 020-039 172 172 172 316 316 316 F F F 2 040-059 177 177 177 326 326 326 F F F 3 060-079 167 167 167 305 305 305 F F F 4 080-099 173 173 173 318 318 318 F F F 5 100-119 176 176 176 324 324 324 F F F 6 120-139 174 174 174 320 320 320 F F F 7 140-159 172 172 172 316 316 316 F F F 8 160-179 178 178 178 328 328 328 F F F 9 180-199 178 178 178 328 328 328 F F F 10 200-219 178 178 178 328 328 328 F F F 11 220-239 176 176 176 323 323 323 F F F 12 240-259 176 176 176 323 323 323 F F F 13 260-279 174 174 174 320 320 320 F F F 14 280-299 179 179 179 329 329 329 F F F 15 300-319 179 179 179 329 329 329 F F F 16 320-339 178 178 178 328 328 328 F F F 17 340-359 178 178 178 328 328 328 F F F 18 360-379 169 169 169 308 308 308 F F F 19 380-399 160 160 160 299 299 299 F F F 20 400-419 166 166 166 304 304 304 F F F 21 420-439 176 176 176 324 324 324 F F F 22 440-459 178 178 178 328 328 328 F F F 23 460-479 177 177 177 326 326 326 F F F 24 480-499 175 175 175 321 321 321 F F F 25 500-519 174 174 174 320 320 320 F F F 26 520-539 179 179 179 329 329 329 F F F 27 540-559 178 178 178 328 328 328 F F F 28 560-579 178 178 178 328 328 328 F F F 29 580-599 175 175 175 321 321 321 F F F Max gen: 179, Max flush: 329 Begin loop 90 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.032912579Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.078015808Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.078592421Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.078627991Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.078643142Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.078658042Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907} {"msg":"Crucible bb6fde5a-6687-4d9f-83e0-4fbd5b1abc3a has session id: 1557cf7d-5e3f-4a70-a3f4-8547777dfb1e","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.079084911Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"Upstairs opts: Upstairs UUID: bb6fde5a-6687-4d9f-83e0-4fbd5b1abc3a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.079118372Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"Crucible stats registered with UUID: bb6fde5a-6687-4d9f-83e0-4fbd5b1abc3a","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.079137642Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.110069233Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.110113404Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907} {"msg":"bb6fde5a-6687-4d9f-83e0-4fbd5b1abc3a active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.110181396Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.110216376Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"client":"0","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.110249957Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"client":"0","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.110270198Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"client":"1","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.110287408Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"client":"1","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.110312549Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"client":"2","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.110345579Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"client":"2","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.11037797Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"":"io task","client":"2","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.110413661Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"":"io task","client":"0","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.110448032Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"":"io task","client":"1","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"ds_connection connected from Ok(127.0.0.1:41691)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.110483622Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"":"io task","client":"1","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"ds_connection connected from Ok(127.0.0.1:49707)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.110604785Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"":"io task","client":"2","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"ds_connection connected from Ok(127.0.0.1:41455)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.110649866Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"":"io task","client":"0","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.11127651Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"client":"2","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.11131695Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"client":"2","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.111345281Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"client":"1","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.111370221Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"client":"1","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.111398652Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"client":"0","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.111427803Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"client":"0","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.111992795Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"client":"2","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.112023646Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"client":"1","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.112039406Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"client":"0","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.112560427Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"client":"2","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.112586968Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"client":"2","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.112601198Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.112614099Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"client":"0","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.112627569Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"client":"0","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.112640859Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.11265672Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"client":"1","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.11267142Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"client":"1","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"[0]R flush_numbers[0..12]: [328, 316, 326, 305, 318, 324, 320, 316, 328, 328, 328, 323]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.11268542Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"":"downstairs","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"[0]R generation[0..12]: [178, 172, 177, 167, 173, 176, 174, 172, 178, 178, 178, 176]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.11269922Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"":"downstairs","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.112713271Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"":"downstairs","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"[1]R flush_numbers[0..12]: [328, 316, 326, 305, 318, 324, 320, 316, 328, 328, 328, 323]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.112727211Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"":"downstairs","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"[1]R generation[0..12]: [178, 172, 177, 167, 173, 176, 174, 172, 178, 178, 178, 176]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.112744421Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"":"downstairs","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.112771842Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"":"downstairs","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"[2]R flush_numbers[0..12]: [328, 316, 326, 305, 318, 324, 320, 316, 328, 328, 328, 323]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.112801672Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"":"downstairs","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"[2]R generation[0..12]: [178, 172, 177, 167, 173, 176, 174, 172, 178, 178, 178, 176]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:56:44.112826113Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"":"downstairs","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.112862394Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"":"downstairs","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.112889365Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"":"downstairs","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.112916075Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"":"downstairs","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"Next flush: 330","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.112941306Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"":"downstairs","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.112959796Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"":"downstairs","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.112982427Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.113006727Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"client":"0","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.113030888Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"client":"1","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.113054308Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"client":"2Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ","":"downstairs","session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.113082759Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.113105419Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"bb6fde5a-6687-4d9f-83e0-4fbd5b1abc3a is now active with session: 1557cf7d-5e3f-4a70-a3f4-8547777dfb1e","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.11312855Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.11315053Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907,"session_id":"1557cf7d-5e3f-4a70-a3f4-8547777dfb1e"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:44.113171911Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 59 len 32768 data: 16 15 14 12 12 12 15 16 02/30 Read block 202 len 24576 03/30 Write block 83 len 20480 data: 5 5 7 7 7 04/30 Read block 221 len 32768 05/30 Read block 354 len 4096 06/30 Read block 347 len 32768 07/30 Write block 206 len 28672 data: 13 10 10 9 14 15 14 08/30 Read block 533 len 24576 09/30 Write block 334 len 36864 data: 15 18 20 18 19 15 14 14 13 10/30 Read block 551 len 16384 11/30 Flush 12/30 Write block 314 len 16384 data: 13 14 11 11 13/30 Write block 298 len 4096 data: 14 14/30 Write block 366 len 36864 data: 12 13 11 11 11 10 10 9 11 15/30 Write block 324 len 8192 data: 9 11 16/30 Read block 12 len 40960 17/30 Write block 96 len 28672 data: 13 11 12 11 11 13 13 18/30 Flush 19/30 Write block 340 len 24576 data: 15 15 14 10 12 12 20/30 Flush 21/30 Write block 427 len 36864 data: 11 14 14 14 15 15 17 16 16 22/30 Read block 256 len 40960 23/30 Write block 519 len 40960 data: 10 10 11 9 13 18 19 20 19 19 24/30 Flush 25/30 Write block 541 len 36864 data: 14 14 13 13 18 17 18 18 15 26/30 Read block 151 len 28672 27/30 Write block 168 len 32768 data: 17 17 13 13 13 13 15 15 28/30 Read block 283 len 40960 29/30 Write block 3 len 24576 data: 8 6 9 10 11 14 30/30 Write block 406 len 16384 data: 13 11 13 15 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 9 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:56:44.134784629Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1907} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 328 328 328 T T T 1 020-039 172 172 172 316 316 316 F F F 2 040-059 180 180 180 330 330 330 F F F 3 060-079 180 180 180 330 330 330 F F F 4 080-099 180 180 180 331 331 331 F F F 5 100-119 180 180 180 331 331 331 F F F 6 120-139 174 174 174 320 320 320 F F F 7 140-159 172 172 172 316 316 316 F F F 8 160-179 178 178 178 328 328 328 T T T 9 180-199 178 178 178 328 328 328 F F F 10 200-219 180 180 180 330 330 330 F F F 11 220-239 176 176 176 323 323 323 F F F 12 240-259 176 176 176 323 323 323 F F F 13 260-279 174 174 174 320 320 320 F F F 14 280-299 180 180 180 331 331 331 F F F 15 300-319 180 180 180 331 331 331 F F F 16 320-339 180 180 180 331 331 331 F F F 17 340-359 180 180 180 332 332 332 F F F 18 360-379 180 180 180 331 331 331 F F F 19 380-399 160 160 160 299 299 299 F F F 20 400-419 166 166 166 304 304 304 T T T 21 420-439 180 180 180 333 333 333 F F F 22 440-459 178 178 178 328 328 328 F F F 23 460-479 177 177 177 326 326 326 F F F 24 480-499 175 175 175 321 321 321 F F F 25 500-519 180 180 180 333 333 333 F F F 26 520-539 180 180 180 333 333 333 F F F 27 540-559 178 178 178 328 328 328 T T T 28 560-579 178 178 178 328 328 328 F F F 29 580-599 175 175 175 321 321 321 F F F Max gen: 180, Max flush: 333 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 334 334 334 F F F 1 020-039 172 172 172 316 316 316 F F F 2 040-059 180 180 180 330 330 330 F F F 3 060-079 180 180 180 330 330 330 F F F 4 080-099 180 180 180 331 331 331 F F F 5 100-119 180 180 180 331 331 331 F F F 6 120-139 174 174 174 320 320 320 F F F 7 140-159 172 172 172 316 316 316 F F F 8 160-179 181 181 181 334 334 334 F F F 9 180-199 178 178 178 328 328 328 F F F 10 200-219 180 180 180 330 330 330 F F F 11 220-239 176 176 176 323 323 323 F F F 12 240-259 176 176 176 323 323 323 F F F 13 260-279 174 174 174 320 320 320 F F F 14 280-299 180 180 180 331 331 331 F F F 15 300-319 180 180 180 331 331 331 F F F 16 320-339 180 180 180 331 331 331 F F F 17 340-359 180 180 180 332 332 332 F F F 18 360-379 180 180 180 331 331 331 F F F 19 380-399 160 160 160 299 299 299 F F F 20 400-419 181 181 181 334 334 334 F F F 21 420-439 180 180 180 333 333 333 F F F 22 440-459 178 178 178 328 328 328 F F F 23 460-479 177 177 177 326 326 326 F F F 24 480-499 175 175 175 321 321 321 F F F 25 500-519 180 180 180 333 333 333 F F F 26 520-539 180 180 180 333 333 333 F F F 27 540-559 181 181 181 334 334 334 F F F 28 560-579 178 178 178 328 328 328 F F F 29 580-599 175 175 175 321 321 321 F F F Max gen: 181, Max flush: 334 Begin loop 91 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.395682661Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.439914881Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.440639196Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.440675787Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.440690468Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.440705068Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915} {"msg":"Crucible 95bbe374-844d-4d2f-bd28-16d3c6fbc615 has session id: 0951de2d-8f07-44e8-bbf8-5c5382c36bc8","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.441121687Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"Upstairs opts: Upstairs UUID: 95bbe374-844d-4d2f-bd28-16d3c6fbc615, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.441159528Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"Crucible stats registered with UUID: 95bbe374-844d-4d2f-bd28-16d3c6fbc615","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.441175448Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.471899355Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.471924555Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915} {"msg":"95bbe374-844d-4d2f-bd28-16d3c6fbc615 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.471987607Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.472026037Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"client":"0","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.472057688Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"client":"0","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.472075829Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"client":"1","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.472093029Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"client":"1","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.472110139Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"client":"2","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.47212693Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"client":"2","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.47214347Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"":"io task","client":"2","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.47216091Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"":"io task","client":"0","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.472177891Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"":"io task","client":"1","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"ds_connection connected from Ok(127.0.0.1:41137)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.472305863Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"":"io task","client":"1","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"ds_connection connected from Ok(127.0.0.1:39259)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.472571719Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"":"io task","client":"0","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"ds_connection connected from Ok(127.0.0.1:32939)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.47262248Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"":"io task","client":"2","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.473102561Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"client":"1","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.473137772Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"client":"1","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.473164382Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"client":"0","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.473196533Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"client":"0","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.473223953Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"client":"2","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.473239674Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"client":"2","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.473795436Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"client":"1","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.473825836Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"client":"0","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.473842207Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"client":"2","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.474371468Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"client":"0","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.474400399Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"client":"0","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.474415929Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.474485231Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"client":"1","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.474505501Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"client":"1","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.474519391Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.474531862Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"client":"2","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.474545382Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"client":"2","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"[0]R flush_numbers[0..12]: [334, 316, 330, 330, 331, 331, 320, 316, 334, 328, 330, 323]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.474559462Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"":"downstairs","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"[0]R generation[0..12]: [181, 172, 180, 180, 180, 180, 174, 172, 181, 178, 180, 176]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.474576703Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"":"downstairs","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.474591403Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"":"downstairs","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"[1]R flush_numbers[0..12]: [334, 316, 330, 330, 331, 331, 320, 316, 334, 328, 330, 323]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.474605423Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"":"downstairs","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"[1]R generation[0..12]: [181, 172, 180, 180, 180, 180, 174, 172, 181, 178, 180, 176]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.474619444Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"":"downstairs","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.474633444Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"":"downstairs","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"[2]R flush_numbers[0..12]: [334, 316, 330, 330, 331, 331, 320, 316, 334, 328, 330, 323]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.474647324Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"":"downstairs","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"[2]R generation[0..12]: [181, 172, 180, 180, 180, 180, 174, 172, 181, 178, 180, 176]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.474661125Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"":"downstairs","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.474674885Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"":"downstairs","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.474687995Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"":"downstairs","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:56:48.474701335Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"":"downstairs","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"Next flush: 335","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.474722716Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"":"downstairs","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.474736036Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"":"downstairs","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.474749556Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.474761827Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"client":"0","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.474775027Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"client":"1","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.474791247Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"client":"2","":"downstairs","session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.474804938Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.474817328Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"95bbe374-844d-4d2f-bd28-16d3c6fbc615 is now active with session: 0951de2d-8f07-44e8-bbf8-5c5382c36bc8","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.474830268Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.474842398Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915,"session_id":"0951de2d-8f07-44e8-bbf8-5c5382c36bc8"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:48.474854789Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 121 len 28672 data: 12 10 14 13 13 14 12 03/30 Write block 564 len 32768 data: 6 6 9 10 9 13 17 17 04/30 Read block 590 len 20480 05/30 Write block 563 len 4096 data: 8 06/30 Write block 286 len 16384 data: 13 11 10 11 07/30 Flush 08/30 Write block 300 len 16384 data: 14 12 12 11 09/30 Flush 10/30 Flush 11/30 Write block 479 len 28672 data: 18 19 17 17 16 15 15 12/30 Read block 320 len 28672 13/30 Read block 2 len 36864 14/30 Write block 155 len 40960 data: 19 15 17 16 16 17 17 16 16 17 15/30 Write block 551 len 16384 data: 15 19 17 15 16/30 Write block 515 len 28672 data: 15 15 14 13 11 11 12 17/30 Write block 446 len 8192 data: 12 13 18/30 Read block 78 len 20480 19/30 Read block 417 len 28672 20/30 Read block 384 len 40960 21/30 Read block 21 len 40960 22/30 Write block 147 len 40960 data: 15 16 17 17 17 19 17 17 20 16 23/30 Read block 306 len 32768 24/30 Write block 122 len 24576 data: 11 15 14 14 15 13 25/30 Read block 196 len 24576 26/30 Flush 27/30 Write block 476 len 36864 data: 19 19 19 19 20 18 18 17 16 28/30 Read block 270 len 24576 29/30 Write block 309 len 28672 data: 10 9 10 11 11 14 15 30/30 Write block 587 len 4096 data: 8 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 9 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:56:48.49473491Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1915} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 334 334 334 F F F 1 020-039 172 172 172 316 316 316 F F F 2 040-059 180 180 180 330 330 330 F F F 3 060-079 180 180 180 330 330 330 F F F 4 080-099 180 180 180 331 331 331 F F F 5 100-119 180 180 180 331 331 331 F F F 6 120-139 182 182 182 339 339 339 F F F 7 140-159 182 182 182 339 339 339 F F F 8 160-179 182 182 182 339 339 339 F F F 9 180-199 178 178 178 328 328 328 F F F 10 200-219 180 180 180 330 330 330 F F F 11 220-239 176 176 176 323 323 323 F F F 12 240-259 176 176 176 323 323 323 F F F 13 260-279 174 174 174 320 320 320 F F F 14 280-299 182 182 182 336 336 336 F F F 15 300-319 182 182 182 337 337 337 T T T 16 320-339 180 180 180 331 331 331 F F F 17 340-359 180 180 180 332 332 332 F F F 18 360-379 180 180 180 331 331 331 F F F 19 380-399 160 160 160 299 299 299 F F F 20 400-419 181 181 181 334 334 334 F F F 21 420-439 180 180 180 333 333 333 F F F 22 440-459 182 182 182 339 339 339 F F F 23 460-479 182 182 182 339 339 339 T T T 24 480-499 182 182 182 339 339 339 T T T 25 500-519 182 182 182 339 339 339 F F F 26 520-539 182 182 182 339 339 339 F F F 27 540-559 182 182 182 339 339 339 F F F 28 560-579 182 182 182 336 336 336 F F F 29 580-599 175 175 175 321 321 321 T T T Max gen: 182, Max flush: 339 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 334 334 334 F F F 1 020-039 172 172 172 316 316 316 F F F 2 040-059 180 180 180 330 330 330 F F F 3 060-079 180 180 180 330 330 330 F F F 4 080-099 180 180 180 331 331 331 F F F 5 100-119 180 180 180 331 331 331 F F F 6 120-139 182 182 182 339 339 339 F F F 7 140-159 182 182 182 339 339 339 F F F 8 160-179 182 182 182 339 339 339 F F F 9 180-199 178 178 178 328 328 328 F F F 10 200-219 180 180 180 330 330 330 F F F 11 220-239 176 176 176 323 323 323 F F F 12 240-259 176 176 176 323 323 323 F F F 13 260-279 174 174 174 320 320 320 F F F 14 280-299 182 182 182 336 336 336 F F F 15 300-319 183 183 183 340 340 340 F F F 16 320-339 180 180 180 331 331 331 F F F 17 340-359 180 180 180 332 332 332 F F F 18 360-379 180 180 180 331 331 331 F F F 19 380-399 160 160 160 299 299 299 F F F 20 400-419 181 181 181 334 334 334 F F F 21 420-439 180 180 180 333 333 333 F F F 22 440-459 182 182 182 339 339 339 F F F 23 460-479 183 183 183 340 340 340 F F F 24 480-499 183 183 183 340 340 340 F F F 25 500-519 182 182 182 339 339 339 F F F 26 520-539 182 182 182 339 339 339 F F F 27 540-559 182 182 182 339 339 339 F F F 28 560-579 182 182 182 336 336 336 F F F 29 580-599 183 183 183 340 340 340 F F F Max gen: 183, Max flush: 340 Begin loop 92 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.753074836Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:56:52.796524558Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.796889816Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.829377171Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.829939223Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.829975314Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.829990604Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.830005725Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923} {"msg":"Crucible ae734ece-940f-4072-8ac0-b4650a00c9f0 has session id: 1140ae70-f263-4fa4-9c77-d123187b8df0","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.830471325Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"Upstairs opts: Upstairs UUID: ae734ece-940f-4072-8ac0-b4650a00c9f0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.830517816Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"Crucible stats registered with UUID: ae734ece-940f-4072-8ac0-b4650a00c9f0","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.830543506Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.862023879Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.86205063Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923} {"msg":"ae734ece-940f-4072-8ac0-b4650a00c9f0 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.862135462Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.862158132Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"client":"0","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.862187563Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"client":"0","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.862209863Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"client":"1","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.862227744Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"client":"1","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.862251424Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"client":"2","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.862313555Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"client":"2","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.862406448Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"":"io task","client":"2","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.862441978Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"":"io task","client":"0","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.862475909Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"":"io task","client":"1","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"ds_connection connected from Ok(127.0.0.1:34358)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.86250703Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"":"io task","client":"2","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"ds_connection connected from Ok(127.0.0.1:45384)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.862614182Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"":"io task","client":"0","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"ds_connection connected from Ok(127.0.0.1:56876)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.862645233Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"":"io task","client":"1","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.863320367Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"client":"2","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.863352418Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"client":"2","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.863370098Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"client":"1","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.863392459Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"client":"1","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.86341933Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"client":"0","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.86346784Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"client":"0","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.864086824Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"client":"1","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.864124135Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"client":"2","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.864190696Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"client":"0","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.864733868Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"client":"1","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.864774199Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"client":"1","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.864803669Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.86481985Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"client":"2","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.86483413Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"client":"2","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.86484805Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.864860721Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"client":"0","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.864874611Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"client":"0","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"[0]R flush_numbers[0..12]: [334, 316, 330, 330, 331, 331, 339, 339, 339, 328, 330, 323]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.864888661Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"":"downstairs","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"[0]R generation[0..12]: [181, 172, 180, 180, 180, 180, 182, 182, 182, 178, 180, 176]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.864902832Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"":"downstairs","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.864916892Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"":"downstairs","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"[1]R flush_numbers[0..12]: [334, 316, 330, 330, 331, 331, 339, 339, 339, 328, 330, 323]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.864934472Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"":"downstairs","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"[1]R generation[0..12]: [181, 172, 180, 180, 180, 180, 182, 182, 182, 178, 180, 176]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.864948553Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"":"downstairs","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.864969863Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"":"downstairs","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"Wait for a query_work_queue command to finish before sending IO msg":"[2]R flush_numbers[0..12]: [334, 316, 330, 330, 331, 331, 339, 339, 339, 328, 330, 323]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.865052865Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"":"downstairs","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"[2]R generation[0..12]: [181, 172, 180, 180, 180, 180, 182, 182, 182, 178, 180, 176]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.865086626Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"":"downstairs","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.865104936Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"":"downstairs","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.865119686Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"":"downstairs","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.865133977Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"":"downstairs","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"Next flush: 341","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.865147487Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"":"downstairs","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.865160887Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"":"downstairs","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.865174257Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.865186458Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"client":"0","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.865200138Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"client":"1","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.865213638Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"client":"2","":"downstairs","session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.865227129Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.865243949Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"ae734ece-940f-4072-8ac0-b4650a00c9f0 is now active with session: 1140ae70-f263-4fa4-9c77-d123187b8df0","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.865257719Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.86527034Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923,"session_id":"1140ae70-f263-4fa4-9c77-d123187b8df0"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 } {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:52.86528734Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 264 len 16384 data: 8 10 14 14 02/30 Flush 03/30 Write block 518 len 4096 data: 14 04/30 Write block 353 len 4096 data: 9 05/30 Read block 355 len 20480 06/30 Write block 227 len 24576 data: 13 11 14 14 15 17 07/30 Read block 61 len 32768 08/30 Read block 379 len 40960 09/30 Read block 252 len 12288 10/30 Read block 255 len 32768 11/30 Read block 253 len 40960 12/30 Read block 521 len 16384 13/30 Read block 580 len 24576 14/30 Read block 181 len 24576 15/30 Write block 548 len 36864 data: 19 16 15 16 20 18 16 16 14 16/30 Write block 296 len 32768 data: 11 12 15 15 15 13 13 12 17/30 Write block 45 len 24576 data: 12 12 10 11 11 15 18/30 Write block 328 len 4096 data: 11 19/30 Write block 261 len 8192 data: 8 8 20/30 Flush 21/30 Write block 384 len 40960 data: 10 8 7 13 13 13 11 12 13 15 22/30 Read block 258 len 4096 23/30 Write block 596 len 8192 data: 6 7 24/30 Flush 25/30 Write block 593 len 8192 data: 8 7 26/30 Read block 200 len 36864 27/30 Read block 102 len 16384 28/30 Read block 449 len 36864 29/30 Write block 575 len 12288 data: 22 16 15 30/30 Write block 444 len 4096 data: 16 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 2 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:56:52.88324466Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1923} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 334 334 334 F F F 1 020-039 172 172 172 316 316 316 F F F 2 040-059 184 184 184 342 342 342 F F F 3 060-079 180 180 180 330 330 330 F F F 4 080-099 180 180 180 331 331 331 F F F 5 100-119 180 180 180 331 331 331 F F F 6 120-139 182 182 182 339 339 339 F F F 7 140-159 182 182 182 339 339 339 F F F 8 160-179 182 182 182 339 339 339 F F F 9 180-199 178 178 178 328 328 328 F F F 10 200-219 180 180 180 330 330 330 F F F 11 220-239 184 184 184 342 342 342 F F F 12 240-259 176 176 176 323 323 323 F F F 13 260-279 184 184 184 342 342 342 F F F 14 280-299 184 184 184 342 342 342 F F F 15 300-319 184 184 184 342 342 342 F F F 16 320-339 184 184 184 342 342 342 F F F 17 340-359 184 184 184 342 342 342 F F F 18 360-379 180 180 180 331 331 331 F F F 19 380-399 184 184 184 343 343 343 F F F 20 400-419 181 181 181 334 334 334 F F F 21 420-439 180 180 180 333 333 333 F F F 22 440-459 182 182 182 339 339 339 T T T 23 460-479 183 183 183 340 340 340 F F F 24 480-499 183 183 183 340 340 340 F F F 25 500-519 184 184 184 342 342 342 F F F 26 520-539 182 182 182 339 339 339 F F F 27 540-559 184 184 184 342 342 342 F F F 28 560-579 182 182 182 336 336 336 T T T 29 580-599 184 184 184 343 343 343 T T T Max gen: 184, Max flush: 343 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 334 334 334 F F F 1 020-039 172 172 172 316 316 316 F F F 2 040-059 184 184 184 342 342 342 F F F 3 060-079 180 180 180 330 330 330 F F F 4 080-099 180 180 180 331 331 331 F F F 5 100-119 180 180 180 331 331 331 F F F 6 120-139 182 182 182 339 339 339 F F F 7 140-159 182 182 182 339 339 339 F F F 8 160-179 182 182 182 339 339 339 F F F 9 180-199 178 178 178 328 328 328 F F F 10 200-219 180 180 180 330 330 330 F F F 11 220-239 184 184 184 342 342 342 F F F 12 240-259 176 176 176 323 323 323 F F F 13 260-279 184 184 184 342 342 342 F F F 14 280-299 184 184 184 342 342 342 F F F 15 300-319 184 184 184 342 342 342 F F F 16 320-339 184 184 184 342 342 342 F F F 17 340-359 184 184 184 342 342 342 F F F 18 360-379 180 180 180 331 331 331 F F F 19 380-399 184 184 184 343 343 343 F F F 20 400-419 181 181 181 334 334 334 F F F 21 420-439 180 180 180 333 333 333 F F F 22 440-459 185 185 185 344 344 344 F F F 23 460-479 183 183 183 340 340 340 F F F 24 480-499 183 183 183 340 340 340 F F F 25 500-519 184 184 184 342 342 342 F F F 26 520-539 182 182 182 339 339 339 F F F 27 540-559 184 184 184 342 342 342 F F F 28 560-579 185 185 185 344 344 344 F F F 29 580-599 185 185 185 344 344 344 F F F Max gen: 185, Max flush: 344 Begin loop 93 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.172630979Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:56:57.215712034Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.216106702Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.247847321Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.248391073Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.248426194Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.248440154Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.248454634Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931} {"msg":"Crucible 81ce0251-f086-4d9b-afba-0d0169b93147 has session id: 5325085a-9d64-4e07-853a-efd09c54787e","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.248913154Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"Upstairs opts: Upstairs UUID: 81ce0251-f086-4d9b-afba-0d0169b93147, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.248958715Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"Crucible stats registered with UUID: 81ce0251-f086-4d9b-afba-0d0169b93147","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.248979935Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.280557491Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.280601621Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931} {"msg":"81ce0251-f086-4d9b-afba-0d0169b93147 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.280668543Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.280702724Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"client":"0","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.280735675Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"client":"0","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.280755335Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"client":"1","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.280786236Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"client":"1","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.280827126Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"client":"2","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.280852357Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"client":"2","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.280869777Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"":"io task","client":"2","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.280900238Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"":"io task","client":"0","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.280934129Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"":"io task","client":"1","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"ds_connection connected from Ok(127.0.0.1:41219)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.28096608Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"":"io task","client":"2","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"ds_connection connected from Ok(127.0.0.1:50767)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.28099916Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"":"io task","client":"0","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"ds_connection connected from Ok(127.0.0.1:33801)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.281095972Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"":"io task","client":"1","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.281694165Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"client":"1","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.281722216Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"client":"1","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.281737466Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"client":"2","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.281751257Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"client":"2","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.281796998Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"client":"0","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.281873089Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"client":"0","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.28236011Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"client":"1","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.282456932Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"client":"0","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.282488053Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"client":"2","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.282944712Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"client":"1","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.282985843Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"client":"1","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.283015224Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.283123246Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"client":"0","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.283153127Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"client":"0","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.283168157Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.283181528Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"client":"2","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.283196468Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"client":"2","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"[0]R flush_numbers[0..12]: [334, 316, 342, 330, 331, 331, 339, 339, 339, 328, 330, 342]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.283210608Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"":"downstairs","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"[0]R generation[0..12]: [181, 172, 184, 180, 180, 180, 182, 182, 182, 178, 180, 184]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.283225429Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"":"downstairs","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.283239919Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"":"downstairs","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"[1]R flush_numbers[0..12]: [334, 316, 342, 330, 331, 331, 339, 339, 339, 328, 330, 342]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.283253799Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"":"downstairs","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"[1]R generation[0..12]: [181, 172, 184, 180, 180, 180, 182, 182, 182, 178, 180, 184]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.283267599Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"":"downstairs","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.28328532Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"":"downstairs","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"[2]R flush_numbers[0..12]: [334, 316, 342, 330, 331, 331, 339, 339, 339, 328, 330, 342]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.28330062Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"":"downstairs","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"[2]R generation[0..12]: [181, 172, 184, 180, 180, 180, 182, 182, 182, 178, 180, 184]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.28331458Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"":"downstairs","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.283328651Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN",Wait for a query_work_queue command to finish before sending IO "pid":1931,"":"downstairs","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.283411953Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"":"downstairs","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.283428513Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"":"downstairs","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"Next flush: 345","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.283441733Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"":"downstairs","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.283454914Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"":"downstairs","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.283468274Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.283480354Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"client":"0","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.283493734Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"client":"1","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.283507035Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"client":"2","":"downstairs","session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.283520295Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.283532555Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"81ce0251-f086-4d9b-afba-0d0169b93147 is now active with session: 5325085a-9d64-4e07-853a-efd09c54787e","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.283545956Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.283560826Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931,"session_id":"5325085a-9d64-4e07-853a-efd09c54787e"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:56:57.283574006Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 472 len 28672 data: 12 12 15 17 20 20 20 03/30 Write block 119 len 12288 data: 11 10 13 04/30 Write block 485 len 12288 data: 16 14 15 05/30 Flush 06/30 Write block 409 len 8192 data: 16 18 07/30 Write block 522 len 4096 data: 10 08/30 Flush 09/30 Write block 459 len 32768 data: 17 16 18 18 18 15 12 13 10/30 Write block 28 len 16384 data: 17 15 11 12 11/30 Write block 109 len 20480 data: 13 15 15 17 17 12/30 Read block 98 len 4096 13/30 Read block 78 len 16384 14/30 Read block 515 len 28672 15/30 Read block 2 len 28672 16/30 Read block 64 len 16384 17/30 Write block 109 len 8192 data: 14 16 18/30 Read block 599 len 4096 19/30 Flush 20/30 Write block 419 len 8192 data: 6 10 21/30 Read block 142 len 8192 22/30 Read block 12 len 32768 23/30 Read block 255 len 24576 24/30 Read block 278 len 24576 25/30 Write block 335 len 28672 data: 19 21 19 20 16 16 16 26/30 Write block 548 len 36864 data: 20 17 16 17 21 19 17 17 15 27/30 Write block 527 len 32768 data: 20 20 20 20 17 17 16 16 28/30 Read block 211 len 20480 29/30 Read block 487 len 32768 30/30 Read block 208 len 40960 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 2 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 10 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 10 11 10 31 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:56:57.30172514Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1931} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 334 334 334 F F F 1 020-039 186 186 186 348 348 348 F F F 2 040-059 184 184 184 342 342 342 F F F 3 060-079 180 180 180 330 330 330 F F F 4 080-099 180 180 180 331 331 331 F F F 5 100-119 186 186 186 348 348 348 F F F 6 120-139 186 186 186 346 346 346 F F F 7 140-159 182 182 182 339 339 339 F F F 8 160-179 182 182 182 339 339 339 F F F 9 180-199 178 178 178 328 328 328 F F F 10 200-219 180 180 180 330 330 330 F F F 11 220-239 184 184 184 342 342 342 F F F 12 240-259 176 176 176 323 323 323 F F F 13 260-279 184 184 184 342 342 342 F F F 14 280-299 184 184 184 342 342 342 F F F 15 300-319 184 184 184 342 342 342 F F F 16 320-339 184 184 184 342 342 342 T T T 17 340-359 184 184 184 342 342 342 T T T 18 360-379 180 180 180 331 331 331 F F F 19 380-399 184 184 184 343 343 343 F F F 20 400-419 186 186 186 347 347 347 T T T 21 420-439 180 180 180 333 333 333 T T T 22 440-459 186 186 186 348 348 348 F F F 23 460-479 186 186 186 348 348 348 F F F 24 480-499 186 186 186 346 346 346 F F F 25 500-519 184 184 184 342 342 342 F F F 26 520-539 186 186 186 347 347 347 T T T 27 540-559 184 184 184 342 342 342 T T T 28 560-579 185 185 185 344 344 344 F F F 29 580-599 185 185 185 344 344 344 F F F Max gen: 186, Max flush: 348 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 334 334 334 F F F 1 020-039 186 186 186 348 348 348 F F F 2 040-059 184 184 184 342 342 342 F F F 3 060-079 180 180 180 330 330 330 F F F 4 080-099 180 180 180 331 331 331 F F F 5 100-119 186 186 186 348 348 348 F F F 6 120-139 186 186 186 346 346 346 F F F 7 140-159 182 182 182 339 339 339 F F F 8 160-179 182 182 182 339 339 339 F F F 9 180-199 178 178 178 328 328 328 F F F 10 200-219 180 180 180 330 330 330 F F F 11 220-239 184 184 184 342 342 342 F F F 12 240-259 176 176 176 323 323 323 F F F 13 260-279 184 184 184 342 342 342 F F F 14 280-299 184 184 184 342 342 342 F F F 15 300-319 184 184 184 342 342 342 F F F 16 320-339 187 187 187 349 349 349 F F F 17 340-359 187 187 187 349 349 349 F F F 18 360-379 180 180 180 331 331 331 F F F 19 380-399 184 184 184 343 343 343 F F F 20 400-419 187 187 187 349 349 349 F F F 21 420-439 187 187 187 349 349 349 F F F 22 440-459 186 186 186 348 348 348 F F F 23 460-479 186 186 186 348 348 348 F F F 24 480-499 186 186 186 346 346 346 F F F 25 500-519 184 184 184 342 342 342 F F F 26 520-539 187 187 187 349 349 349 F F F 27 540-559 187 187 187 349 349 349 F F F 28 560-579 185 185 185 344 344 344 F F F 29 580-599 185 185 185 344 344 344 F F F Max gen: 187, Max flush: 349 Begin loop 94 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.609262693Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:57:01.65196123Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.652364138Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.683946303Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.684525336Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.684565217Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.684580007Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.684594597Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939} {"msg":"Crucible 484b27a5-f7d1-4e65-a501-79ce1055f030 has session id: 72106124-7653-494e-827b-d6ebd40f79ea","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.685028047Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"Upstairs opts: Upstairs UUID: 484b27a5-f7d1-4e65-a501-79ce1055f030, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.685076658Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"Crucible stats registered with UUID: 484b27a5-f7d1-4e65-a501-79ce1055f030","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.685101208Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.715794364Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.715821385Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939} {"msg":"484b27a5-f7d1-4e65-a501-79ce1055f030 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.715904167Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.715922497Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"client":"0","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.715946318Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"client":"0","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.715960528Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"client":"1","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.715974308Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"client":"1","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.715987809Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"client":"2","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.716013269Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"client":"2","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.71604069Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"":"io task","client":"2","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.71606843Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"":"io task","client":"0","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.716097131Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"":"io task","client":"1","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"ds_connection connected from Ok(127.0.0.1:49036)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.716205853Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"":"io task","client":"2","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"ds_connection connected from Ok(127.0.0.1:56423)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.716326866Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"":"io task","client":"0","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"ds_connection connected from Ok(127.0.0.1:40110)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.716378347Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"":"io task","client":"1","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.717023911Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"client":"2","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.717066312Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"client":"2","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.717098593Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"client":"1","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.717129903Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"client":"1","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.717161184Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"client":"0","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.717180064Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"client":"0","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.717629414Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"client":"2","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.717741807Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"client":"1","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.717766377Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"client":"0","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.718223687Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"client":"2","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.718264578Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"client":"2","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.718294699Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.71836633Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"client":"1","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.718396721Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"client":"1","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.718413861Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.718427182Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"client":"0","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.718441132Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"client":"0","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"[0]R flush_numbers[0..12]: [334, 348, 342, 330, 331, 348, 346, 339, 339, 328, 330, 342]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.718455382Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"":"downstairs","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"[0]R generation[0..12]: [181, 186, 184, 180, 180, 186, 186, 182, 182, 178, 180, 184]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.718469682Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"":"downstairs","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.718483903Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"":"downstairs","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"[1]R flush_numbers[0..12]: [334, 348, 342, 330, 331, 348, 346, 339, 339, 328, 330, 342]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.718497913Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"":"downstairs","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"[1]R generation[0..12]: [181, 186, 184, 180, 180, 186, 186, 182, 182, 178, 180, 184]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.718511833Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"":"downstairs","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.718525944Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"":"downstairs","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"[2]R flush_numbers[0..12]: [334, 348, 342, 330, 331, 348, 346, 339, 339, 328, 330, 342]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.718543764Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"":"downstairs","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"[2]R generation[0..12]: [181, 186, 184, 180, 180, 186, 186, 182, 182, 178, 180, 184]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.718558914Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"":"downstairs","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.718574295Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"":"downstairs","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"Wait for a query_work_queue command to finish before sending IO msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.718595305Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"":"downstairs","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.718609535Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"":"downstairs","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"Next flush: 350","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.718622936Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"":"downstairs","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.718636326Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"":"downstairs","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.718649796Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.718662087Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"client":"0","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.718675617Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"client":"1","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.718688947Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"client":"2","":"downstairs","session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.718705757Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.718718298Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"484b27a5-f7d1-4e65-a501-79ce1055f030 is now active with session: 72106124-7653-494e-827b-d6ebd40f79ea","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.718731428Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.718743598Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939,"session_id":"72106124-7653-494e-827b-d6ebd40f79ea"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:01.718758869Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 121 len 16384 data: 14 12 16 15 02/30 Write block 165 len 4096 data: 14 03/30 Write block 264 len 12288 data: 9 11 15 04/30 Read block 141 len 32768 05/30 Write block 380 len 16384 data: 19 16 13 11 06/30 Write block 383 len 40960 data: 12 11 9 8 14 14 14 12 13 14 07/30 Write block 420 len 32768 data: 11 13 15 14 12 11 12 12 08/30 Write block 546 len 40960 data: 18 19 21 18 17 18 22 20 18 18 09/30 Flush 10/30 Write block 313 len 12288 data: 12 15 16 11/30 Write block 223 len 36864 data: 17 15 13 11 14 12 15 15 16 12/30 Write block 67 len 40960 data: 19 18 18 16 21 18 18 20 19 14 13/30 Read block 179 len 12288 14/30 Write block 226 len 20480 data: 12 15 13 16 16 15/30 Read block 151 len 8192 16/30 Read block 479 len 12288 17/30 Read block 408 len 4096 18/30 Read block 2 len 24576 19/30 Write block 322 len 40960 data: 10 12 10 12 12 14 12 9 11 13 20/30 Read block 170 len 4096 21/30 Read block 305 len 16384 22/30 Write block 474 len 20480 data: 16 18 21 21 21 23/30 Read block 426 len 16384 24/30 Write block 535 len 40960 data: 13 11 14 19 18 14 15 15 14 14 25/30 Read block 343 len 36864 26/30 Write block 404 len 20480 data: 14 13 14 12 14 27/30 Write block 375 len 16384 data: 14 19 19 19 28/30 Write block 483 len 40960 data: 18 17 17 15 16 13 13 16 15 14 29/30 Read block 270 len 16384 30/30 Write block 223 len 4096 data: 18 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 3 Done Done Done false 1010 Acked Write 9 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Read 3 Done Done Done false 1013 Acked Write 5 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 10 Done Done Sent false 1028 Acked Read 4 Done Done Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 3 5 Done 20 20 18 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1008 1008 1008 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:57:01.737847303Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1939} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 334 334 334 F F F 1 020-039 186 186 186 348 348 348 F F F 2 040-059 184 184 184 342 342 342 F F F 3 060-079 180 180 180 330 330 330 T T T 4 080-099 180 180 180 331 331 331 F F F 5 100-119 186 186 186 348 348 348 F F F 6 120-139 188 188 188 350 350 350 F F F 7 140-159 182 182 182 339 339 339 F F F 8 160-179 188 188 188 350 350 350 F F F 9 180-199 178 178 178 328 328 328 F F F 10 200-219 180 180 180 330 330 330 F F F 11 220-239 184 184 184 342 342 342 T T T 12 240-259 176 176 176 323 323 323 F F F 13 260-279 188 188 188 350 350 350 F F F 14 280-299 184 184 184 342 342 342 F F F 15 300-319 184 184 184 342 342 342 T T T 16 320-339 187 187 187 349 349 349 T T T 17 340-359 187 187 187 349 349 349 F F F 18 360-379 180 180 180 331 331 331 T T T 19 380-399 188 188 188 350 350 350 F F F 20 400-419 187 187 187 349 349 349 T T T 21 420-439 188 188 188 350 350 350 F F F 22 440-459 186 186 186 348 348 348 F F F 23 460-479 186 186 186 348 348 348 T T T 24 480-499 186 186 186 346 346 346 T T T 25 500-519 184 184 184 342 342 342 F F F 26 520-539 187 187 187 349 349 349 T T T 27 540-559 188 188 188 350 350 350 T T T 28 560-579 185 185 185 344 344 344 F F F 29 580-599 185 185 185 344 344 344 F F F Max gen: 188, Max flush: 350 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 334 334 334 F F F 1 020-039 186 186 186 348 348 348 F F F 2 040-059 184 184 184 342 342 342 F F F 3 060-079 189 189 189 351 351 351 F F F 4 080-099 180 180 180 331 331 331 F F F 5 100-119 186 186 186 348 348 348 F F F 6 120-139 188 188 188 350 350 350 F F F 7 140-159 182 182 182 339 339 339 F F F 8 160-179 188 188 188 350 350 350 F F F 9 180-199 178 178 178 328 328 328 F F F 10 200-219 180 180 180 330 330 330 F F F 11 220-239 189 189 189 351 351 351 F F F 12 240-259 176 176 176 323 323 323 F F F 13 260-279 188 188 188 350 350 350 F F F 14 280-299 184 184 184 342 342 342 F F F 15 300-319 189 189 189 351 351 351 F F F 16 320-339 189 189 189 351 351 351 F F F 17 340-359 187 187 187 349 349 349 F F F 18 360-379 189 189 189 351 351 351 F F F 19 380-399 188 188 188 350 350 350 F F F 20 400-419 189 189 189 351 351 351 F F F 21 420-439 188 188 188 350 350 350 F F F 22 440-459 186 186 186 348 348 348 F F F 23 460-479 189 189 189 351 351 351 F F F 24 480-499 189 189 189 351 351 351 F F F 25 500-519 184 184 184 342 342 342 F F F 26 520-539 189 189 189 351 351 351 F F F 27 540-559 189 189 189 351 351 351 F F F 28 560-579 185 185 185 344 344 344 F F F 29 580-599 185 185 185 344 344 344 F F F Max gen: 189, Max flush: 351 Begin loop 95 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.064080112Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.108015725Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.108637508Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.108673479Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.108687349Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.108702569Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947} {"msg":"Crucible 11a81b86-4fe5-476c-b59c-38d7e7c04f8b has session id: 895df1bc-b922-49f5-b561-24f47a9bfef2","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.109237231Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"Upstairs opts: Upstairs UUID: 11a81b86-4fe5-476c-b59c-38d7e7c04f8b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.109284962Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"Crucible stats registered with UUID: 11a81b86-4fe5-476c-b59c-38d7e7c04f8b","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.109304793Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.140092021Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.140124451Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947} {"msg":"11a81b86-4fe5-476c-b59c-38d7e7c04f8b active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.140194343Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.140216793Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"client":"0","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.140246784Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"client":"0","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.140266124Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"client":"1","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.140283794Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"client":"1","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.140302285Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"client":"2","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.140339316Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"client":"2","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.140373447Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"":"io task","client":"2","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.140407797Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"":"io task","client":"0","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.140444468Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"":"io task","client":"1","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"ds_connection connected from Ok(127.0.0.1:33657)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.140480719Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"":"io task","client":"2","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"ds_connection connected from Ok(127.0.0.1:40140)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.140507079Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"":"io task","client":"1","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"ds_connection connected from Ok(127.0.0.1:61815)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.14053399Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"":"io task","client":"0","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.141237195Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"client":"0","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.141273566Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"client":"0","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.141323397Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"client":"1","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.141346308Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"client":"1","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.141362198Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"client":"2","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.141377208Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"client":"2","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.14190973Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"client":"0","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.141947471Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"client":"1","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.142131005Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"client":"2","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.142474042Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"client":"0","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.142501863Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"client":"0","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.142516813Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.142615945Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"client":"1","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.142636426Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"client":"1","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.142650246Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.142711457Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"client":"2","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.142730218Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"client":"2","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"[0]R flush_numbers[0..12]: [334, 348, 342, 351, 331, 348, 350, 339, 350, 328, 330, 351]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.142744558Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"":"downstairs","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"[0]R generation[0..12]: [181, 186, 184, 189, 180, 186, 188, 182, 188, 178, 180, 189]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.142758888Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"":"downstairs","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.142773239Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"":"downstairs","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"[1]R flush_numbers[0..12]: [334, 348, 342, 351, 331, 348, 350, 339, 350, 328, 330, 351]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.142787299Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"":"downstairs","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"[1]R generation[0..12]: [181, 186, 184, 189, 180, 186, 188, 182, 188, 178, 180, 189]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.142801229Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"":"downstairs","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.14281934Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"":"downstairs","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"[2]R flush_numbers[0..12]: [334, 348, 342, 351, 331, 348, 350, 339, 350, 328, 330, 351]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.142834Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"":"downstairs","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"[2]R generation[0..12]: [181, 186, 184, 189, 180, 186, 188, 182, 188, 178, 180, 189]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.14284827Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"":"downstairs","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.14286241Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"":"downstairs","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.142875611Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"":"downstairs","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.142889181Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"":"downstairs","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {Wait for a query_work_queue command to finish before sending IO "msg":"Next flush: 352","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.142910231Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"":"downstairs","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.142923842Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"":"downstairs","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.142937402Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.142950023Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"client":"0","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.142963553Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"client":"1","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.142976913Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"client":"2","":"downstairs","session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.142990443Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.143003053Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"11a81b86-4fe5-476c-b59c-38d7e7c04f8b is now active with session: 895df1bc-b922-49f5-b561-24f47a9bfef2","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.143016134Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:06.143031204Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947,"session_id":"895df1bc-b922-49f5-b561-24f47a9bfef2"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-28T04:57:06.143044265Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 177 len 20480 data: 11 11 10 10 8 02/30 Flush 03/30 Write block 179 len 20480 data: 11 11 9 11 15 04/30 Write block 179 len 4096 data: 12 05/30 Write block 161 len 40960 data: 18 17 17 18 15 16 14 18 18 14 06/30 Flush 07/30 Write block 530 len 32768 data: 21 18 18 17 17 14 12 15 08/30 Write block 385 len 24576 data: 10 9 15 15 15 13 09/30 Write block 465 len 8192 data: 13 14 10/30 Read block 342 len 12288 11/30 Flush 12/30 Write block 5 len 28672 data: 10 11 12 15 15 12 15 13/30 Write block 237 len 36864 data: 16 14 10 14 12 11 14 12 14 14/30 Read block 334 len 32768 15/30 Read block 94 len 40960 16/30 Write block 220 len 4096 data: 10 17/30 Read block 261 len 28672 18/30 Write block 519 len 28672 data: 12 12 13 11 14 19 20 19/30 Write block 542 len 36864 data: 16 15 15 19 19 20 22 19 18 20/30 Read block 320 len 20480 21/30 Write block 592 len 20480 data: 10 9 8 6 7 22/30 Read block 121 len 36864 23/30 Read block 203 len 16384 24/30 Read block 484 len 12288 25/30 Write block 266 len 12288 data: 16 15 14 26/30 Read block 465 len 28672 27/30 Write block 21 len 40960 data: 21 18 20 19 18 20 19 18 16 12 28/30 Read block 223 len 24576 29/30 Read block 172 len 32768 30/30 Read block 208 len 4096 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 7 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 19 18 18 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:57:06.163325225Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1947} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 334 334 334 T T T 1 020-039 186 186 186 348 348 348 T T T 2 040-059 184 184 184 342 342 342 F F F 3 060-079 189 189 189 351 351 351 F F F 4 080-099 180 180 180 331 331 331 F F F 5 100-119 186 186 186 348 348 348 F F F 6 120-139 188 188 188 350 350 350 F F F 7 140-159 182 182 182 339 339 339 F F F 8 160-179 190 190 190 353 353 353 F F F 9 180-199 190 190 190 353 353 353 F F F 10 200-219 180 180 180 330 330 330 F F F 11 220-239 189 189 189 351 351 351 T T T 12 240-259 176 176 176 323 323 323 T T T 13 260-279 188 188 188 350 350 350 T T T 14 280-299 184 184 184 342 342 342 F F F 15 300-319 189 189 189 351 351 351 F F F 16 320-339 189 189 189 351 351 351 F F F 17 340-359 187 187 187 349 349 349 F F F 18 360-379 189 189 189 351 351 351 F F F 19 380-399 190 190 190 354 354 354 F F F 20 400-419 189 189 189 351 351 351 F F F 21 420-439 188 188 188 350 350 350 F F F 22 440-459 186 186 186 348 348 348 F F F 23 460-479 190 190 190 354 354 354 F F F 24 480-499 189 189 189 351 351 351 F F F 25 500-519 184 184 184 342 342 342 T T T 26 520-539 190 190 190 354 354 354 T T T 27 540-559 189 189 189 351 351 351 T T T 28 560-579 185 185 185 344 344 344 F F F 29 580-599 185 185 185 344 344 344 T T T Max gen: 190, Max flush: 354 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 355 355 355 F F F 1 020-039 191 191 191 355 355 355 F F F 2 040-059 184 184 184 342 342 342 F F F 3 060-079 189 189 189 351 351 351 F F F 4 080-099 180 180 180 331 331 331 F F F 5 100-119 186 186 186 348 348 348 F F F 6 120-139 188 188 188 350 350 350 F F F 7 140-159 182 182 182 339 339 339 F F F 8 160-179 190 190 190 353 353 353 F F F 9 180-199 190 190 190 353 353 353 F F F 10 200-219 180 180 180 330 330 330 F F F 11 220-239 191 191 191 355 355 355 F F F 12 240-259 191 191 191 355 355 355 F F F 13 260-279 191 191 191 355 355 355 F F F 14 280-299 184 184 184 342 342 342 F F F 15 300-319 189 189 189 351 351 351 F F F 16 320-339 189 189 189 351 351 351 F F F 17 340-359 187 187 187 349 349 349 F F F 18 360-379 189 189 189 351 351 351 F F F 19 380-399 190 190 190 354 354 354 F F F 20 400-419 189 189 189 351 351 351 F F F 21 420-439 188 188 188 350 350 350 F F F 22 440-459 186 186 186 348 348 348 F F F 23 460-479 190 190 190 354 354 354 F F F 24 480-499 189 189 189 351 351 351 F F F 25 500-519 191 191 191 355 355 355 F F F 26 520-539 191 191 191 355 355 355 F F F 27 540-559 191 191 191 355 355 355 F F F 28 560-579 185 185 185 344 344 344 F F F 29 580-599 191 191 191 355 355 355 F F F Max gen: 191, Max flush: 355 Begin loop 96 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.45251673Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.49676982Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.49768877Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.497728801Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.497743661Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.497758941Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955} {"msg":"Crucible bc2f5d1e-b4dd-4170-adbb-8ba47e020845 has session id: 8d3ce93e-7669-4b67-8ec2-51444676189a","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.498315033Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"Upstairs opts: Upstairs UUID: bc2f5d1e-b4dd-4170-adbb-8ba47e020845, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.498348504Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"Crucible stats registered with UUID: bc2f5d1e-b4dd-4170-adbb-8ba47e020845","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.498364285Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.529146192Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.529178663Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955} {"msg":"bc2f5d1e-b4dd-4170-adbb-8ba47e020845 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.529246864Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.529270385Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"client":"0","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.529308186Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"client":"0","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.529343727Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"client":"1","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.529378607Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"client":"1","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.529414558Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"client":"2","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.529448069Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"client":"2","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.52948072Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"":"io task","client":"2","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.52951911Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"":"io task","client":"0","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.529573311Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"":"io task","client":"1","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"ds_connection connected from Ok(127.0.0.1:43948)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.529610923Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"":"io task","client":"2","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"ds_connection connected from Ok(127.0.0.1:54057)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.529638833Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"":"io task","client":"0","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"ds_connection connected from Ok(127.0.0.1:58187)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.529677254Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"":"io task","client":"1","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.530304378Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"client":"0","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.530335798Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"client":"0","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.530362599Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"client":"1","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.53043149Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"client":"1","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.530459791Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"client":"2","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.530486521Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"client":"2","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.530965662Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"client":"1","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.530997502Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"client":"0","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.531130925Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"client":"2","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.531515824Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"client":"0","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.531545814Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"client":"0","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.531562375Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.531651057Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"client":"1","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.531671017Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"client":"1","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.531685497Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.531698818Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"client":"2","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.531713168Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"client":"2","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"[0]R flush_numbers[0..12]: [355, 355, 342, 351, 331, 348, 350, 339, 353, 353, 330, 355]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.531727908Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"":"downstairs","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"[0]R generation[0..12]: [191, 191, 184, 189, 180, 186, 188, 182, 190, 190, 180, 191]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.531742049Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"":"downstairs","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.531756219Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"":"downstairs","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"[1]R flush_numbers[0..12]: [355, 355, 342, 351, 331, 348, 350, 339, 353, 353, 330, 355]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.531770219Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"":"downstairs","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"[1]R generation[0..12]: [191, 191, 184, 189, 180, 186, 188, 182, 190, 190, 180, 191]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.53178405Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"":"downstairs","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.53179943Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"":"downstairs","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"[2]R flush_numbers[0..12]: [355, 355, 342, 351, 331, 348, 350, 339, 353, 353, 330, 355]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.53181784Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"":"downstairs","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"[2]R generation[0..12]: [191, 191, 184, 189, 180, 186, 188, 182, 190, 190, 180, 191]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.531832341Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"":"downstairs","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.531847071Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"":"downstairs","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.531860341Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"":"downstairs","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.531874492Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"":"downstairs","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"Next flush: 356","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.531887552Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"":"downstairs","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:57:10.531900622Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"":"downstairs","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.531924593Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.531937553Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"client":"0","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.531951953Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"client":"1","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.531965443Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"client":"2","":"downstairs","session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.531978954Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.531991644Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"bc2f5d1e-b4dd-4170-adbb-8ba47e020845 is now active with session: 8d3ce93e-7669-4b67-8ec2-51444676189a","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.532005424Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.532017555Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955,"session_id":"8d3ce93e-7669-4b67-8ec2-51444676189a"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:10.532032885Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 241 len 4096 data: 13 02/30 Flush 03/30 Write block 582 len 24576 data: 10 14 14 11 10 9 04/30 Write block 427 len 16384 data: 13 15 15 15 05/30 Write block 242 len 24576 data: 12 15 13 15 17 19 06/30 Read block 316 len 32768 07/30 Write block 182 len 24576 data: 12 16 18 18 19 18 08/30 Read block 493 len 24576 09/30 Read block 568 len 40960 10/30 Read block 456 len 8192 11/30 Write block 198 len 24576 data: 16 13 13 13 17 15 12/30 Flush 13/30 Write block 439 len 16384 data: 13 15 17 15 14/30 Read block 109 len 40960 15/30 Read block 3 len 24576 16/30 Read block 556 len 8192 17/30 Flush 18/30 Write block 353 len 8192 data: 10 8 19/30 Flush 20/30 Write block 52 len 20480 data: 17 22 20 21 21 21/30 Write block 430 len 20480 data: 16 16 16 18 17 22/30 Write block 249 len 20480 data: 19 16 14 14 13 23/30 Write block 44 len 4096 data: 12 24/30 Write block 309 len 12288 data: 11 10 11 25/30 Flush 26/30 Write block 579 len 28672 data: 14 13 12 11 15 15 12 27/30 Read block 161 len 40960 28/30 Read block 10 len 40960 29/30 Read block 349 len 24576 30/30 Read block 422 len 12288 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 7 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 3 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 4 5 5 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:57:10.550589927Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1955} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 355 355 355 F F F 1 020-039 191 191 191 355 355 355 F F F 2 040-059 192 192 192 360 360 360 F F F 3 060-079 189 189 189 351 351 351 F F F 4 080-099 180 180 180 331 331 331 F F F 5 100-119 186 186 186 348 348 348 F F F 6 120-139 188 188 188 350 350 350 F F F 7 140-159 182 182 182 339 339 339 F F F 8 160-179 190 190 190 353 353 353 F F F 9 180-199 192 192 192 357 357 357 F F F 10 200-219 192 192 192 357 357 357 F F F 11 220-239 191 191 191 355 355 355 F F F 12 240-259 192 192 192 360 360 360 F F F 13 260-279 191 191 191 355 355 355 F F F 14 280-299 184 184 184 342 342 342 F F F 15 300-319 192 192 192 360 360 360 F F F 16 320-339 189 189 189 351 351 351 F F F 17 340-359 192 192 192 359 359 359 F F F 18 360-379 189 189 189 351 351 351 F F F 19 380-399 190 190 190 354 354 354 F F F 20 400-419 189 189 189 351 351 351 F F F 21 420-439 192 192 192 360 360 360 F F F 22 440-459 192 192 192 358 358 358 F F F 23 460-479 190 190 190 354 354 354 F F F 24 480-499 189 189 189 351 351 351 F F F 25 500-519 191 191 191 355 355 355 F F F 26 520-539 191 191 191 355 355 355 F F F 27 540-559 191 191 191 355 355 355 F F F 28 560-579 185 185 185 344 344 344 T T T 29 580-599 192 192 192 357 357 357 T T T Max gen: 192, Max flush: 360 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 355 355 355 F F F 1 020-039 191 191 191 355 355 355 F F F 2 040-059 192 192 192 360 360 360 F F F 3 060-079 189 189 189 351 351 351 F F F 4 080-099 180 180 180 331 331 331 F F F 5 100-119 186 186 186 348 348 348 F F F 6 120-139 188 188 188 350 350 350 F F F 7 140-159 182 182 182 339 339 339 F F F 8 160-179 190 190 190 353 353 353 F F F 9 180-199 192 192 192 357 357 357 F F F 10 200-219 192 192 192 357 357 357 F F F 11 220-239 191 191 191 355 355 355 F F F 12 240-259 192 192 192 360 360 360 F F F 13 260-279 191 191 191 355 355 355 F F F 14 280-299 184 184 184 342 342 342 F F F 15 300-319 192 192 192 360 360 360 F F F 16 320-339 189 189 189 351 351 351 F F F 17 340-359 192 192 192 359 359 359 F F F 18 360-379 189 189 189 351 351 351 F F F 19 380-399 190 190 190 354 354 354 F F F 20 400-419 189 189 189 351 351 351 F F F 21 420-439 192 192 192 360 360 360 F F F 22 440-459 192 192 192 358 358 358 F F F 23 460-479 190 190 190 354 354 354 F F F 24 480-499 189 189 189 351 351 351 F F F 25 500-519 191 191 191 355 355 355 F F F 26 520-539 191 191 191 355 355 355 F F F 27 540-559 191 191 191 355 355 355 F F F 28 560-579 193 193 193 361 361 361 F F F 29 580-599 193 193 193 361 361 361 F F F Max gen: 193, Max flush: 361 Begin loop 97 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.79725612Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:57:14.84058554Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.840945098Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.873365411Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.873917633Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.873952314Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.873971374Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.873986585Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963} {"msg":"Crucible efec376b-185d-4ad9-bdd8-05c0d04ae2a8 has session id: f193facf-dbf6-4475-9439-2f8273614933","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.874369533Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"Upstairs opts: Upstairs UUID: efec376b-185d-4ad9-bdd8-05c0d04ae2a8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.874412924Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"Crucible stats registered with UUID: efec376b-185d-4ad9-bdd8-05c0d04ae2a8","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.874430654Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.905006868Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.905033848Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963} {"msg":"efec376b-185d-4ad9-bdd8-05c0d04ae2a8 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.90509632Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.90511558Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"client":"0","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.90513929Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"client":"0","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.905154451Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"client":"1","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.905168371Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"client":"1","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.905189871Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"client":"2","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.905216712Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"client":"2","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.905242423Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"":"io task","client":"2","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.905258983Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"":"io task","client":"0","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.905276513Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"":"io task","client":"1","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"ds_connection connected from Ok(127.0.0.1:47418)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.905646372Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"":"io task","client":"2","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"ds_connection connected from Ok(127.0.0.1:46691)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.905690862Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"":"io task","client":"1","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"ds_connection connected from Ok(127.0.0.1:60808)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.905710623Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"":"io task","client":"0","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.906294266Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"client":"2","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.906322296Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"client":"2","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.906336836Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"client":"1","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.906350527Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"client":"1","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.906373457Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"client":"0","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.906396668Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"client":"0","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.907038682Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"client":"1","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.907204445Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"client":"0","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.907240626Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"client":"2","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.907642725Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"client":"1","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.907676996Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"client":"1","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.907693616Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.907786898Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"client":"0","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.907816188Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"client":"0","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.907831389Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.907843959Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"client":"2","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.90785764Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"client":"2","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"[0]R flush_numbers[0..12]: [355, 355, 360, 351, 331, 348, 350, 339, 353, 357, 357, 355]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.90787187Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"":"downstairs","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"[0]R generation[0..12]: [191, 191, 192, 189, 180, 186, 188, 182, 190, 192, 192, 191]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.90788607Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"":"downstairs","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.90790018Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"":"downstairs","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"[1]R flush_numbers[0..12]: [355, 355, 360, 351, 331, 348, 350, 339, 353, 357, 357, 355]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.907914091Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"":"downstairs","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"[1]R generation[0..12]: [191, 191, 192, 189, 180, 186, 188, 182, 190, 192, 192, 191]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.907927841Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"":"downstairs","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.907941701Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"":"downstairs","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"[2]R flush_numbers[0..12]: [355, 355, 360, 351, 331, 348, 350, 339, 353, 357, 357, 355]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.907966522Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"":"downstairs","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"[2]R generation[0..12]: [191, 191, 192, 189, 180, 186, 188, 182, 190, 192, 192, 191]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.907996163Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"":"downstairs","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-28T04:57:14.908018613Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"":"downstairs","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.908040544Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"":"downstairs","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.908053964Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"":"downstairs","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"Next flush: 362","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.908067134Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"":"downstairs","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.908080284Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"":"downstairs","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.908093754Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.908105875Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"client":"0","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.908119485Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"client":"1","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.908133086Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"client":"2","":"downstairs","session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.908146766Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.908159266Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"efec376b-185d-4ad9-bdd8-05c0d04ae2a8 is now active with session: f193facf-dbf6-4475-9439-2f8273614933","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.908172126Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.908184336Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963,"session_id":"f193facf-dbf6-4475-9439-2f8273614933"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:14.908196537Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 90 len 16384 data: 10 13 14 14 02/30 Write block 409 len 16384 data: 17 19 15 13 03/30 Read block 368 len 16384 04/30 Read block 420 len 32768 05/30 Write block 355 len 20480 data: 11 11 11 13 13 06/30 Write block 179 len 4096 data: 13 07/30 Read block 392 len 8192 08/30 Write block 179 len 32768 data: 14 12 10 13 17 19 19 20 09/30 Read block 125 len 32768 10/30 Read block 269 len 32768 11/30 Flush 12/30 Write block 226 len 28672 data: 13 16 14 17 17 17 18 13/30 Write block 556 len 4096 data: 16 14/30 Read block 333 len 8192 15/30 Read block 253 len 4096 16/30 Flush 17/30 Write block 324 len 40960 data: 11 13 13 15 13 10 12 14 12 13 18/30 Read block 535 len 8192 19/30 Read block 461 len 40960 20/30 Read block 123 len 16384 21/30 Read block 374 len 4096 22/30 Read block 567 len 40960 23/30 Read block 346 len 8192 24/30 Read block 498 len 8192 25/30 Write block 78 len 24576 data: 12 12 10 12 10 6 26/30 Write block 224 len 24576 data: 16 14 14 17 15 18 27/30 Read block 488 len 40960 28/30 Read block 64 len 36864 29/30 Write block 514 len 20480 data: 15 16 16 15 15 30/30 Write block 574 len 24576 data: 22 23 17 16 15 15 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 10 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 5 Sent Done Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 1 2 5 Done 12 13 12 37 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:57:14.927482655Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1963} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 355 355 355 F F F 1 020-039 191 191 191 355 355 355 F F F 2 040-059 192 192 192 360 360 360 F F F 3 060-079 189 189 189 351 351 351 T T T 4 080-099 194 194 194 362 362 362 T T T 5 100-119 186 186 186 348 348 348 F F F 6 120-139 188 188 188 350 350 350 F F F 7 140-159 182 182 182 339 339 339 F F F 8 160-179 194 194 194 362 362 362 F F F 9 180-199 194 194 194 362 362 362 F F F 10 200-219 192 192 192 357 357 357 F F F 11 220-239 194 194 194 363 363 363 T T T 12 240-259 192 192 192 360 360 360 F F F 13 260-279 191 191 191 355 355 355 F F F 14 280-299 184 184 184 342 342 342 F F F 15 300-319 192 192 192 360 360 360 F F F 16 320-339 189 189 189 351 351 351 T T T 17 340-359 194 194 194 362 362 362 F F F 18 360-379 189 189 189 351 351 351 F F F 19 380-399 190 190 190 354 354 354 F F F 20 400-419 194 194 194 362 362 362 F F F 21 420-439 192 192 192 360 360 360 F F F 22 440-459 192 192 192 358 358 358 F F F 23 460-479 190 190 190 354 354 354 F F F 24 480-499 189 189 189 351 351 351 F F F 25 500-519 191 191 191 355 355 355 T T T 26 520-539 191 191 191 355 355 355 F F F 27 540-559 194 194 194 363 363 363 F F F 28 560-579 193 193 193 361 361 361 T T T 29 580-599 193 193 193 361 361 361 F F F Max gen: 194, Max flush: 363 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 355 355 355 F F F 1 020-039 191 191 191 355 355 355 F F F 2 040-059 192 192 192 360 360 360 F F F 3 060-079 195 195 195 364 364 364 F F F 4 080-099 195 195 195 364 364 364 F F F 5 100-119 186 186 186 348 348 348 F F F 6 120-139 188 188 188 350 350 350 F F F 7 140-159 182 182 182 339 339 339 F F F 8 160-179 194 194 194 362 362 362 F F F 9 180-199 194 194 194 362 362 362 F F F 10 200-219 192 192 192 357 357 357 F F F 11 220-239 195 195 195 364 364 364 F F F 12 240-259 192 192 192 360 360 360 F F F 13 260-279 191 191 191 355 355 355 F F F 14 280-299 184 184 184 342 342 342 F F F 15 300-319 192 192 192 360 360 360 F F F 16 320-339 195 195 195 364 364 364 F F F 17 340-359 194 194 194 362 362 362 F F F 18 360-379 189 189 189 351 351 351 F F F 19 380-399 190 190 190 354 354 354 F F F 20 400-419 194 194 194 362 362 362 F F F 21 420-439 192 192 192 360 360 360 F F F 22 440-459 192 192 192 358 358 358 F F F 23 460-479 190 190 190 354 354 354 F F F 24 480-499 189 189 189 351 351 351 F F F 25 500-519 195 195 195 364 364 364 F F F 26 520-539 191 191 191 355 355 355 F F F 27 540-559 194 194 194 363 363 363 F F F 28 560-579 195 195 195 364 364 364 F F F 29 580-599 193 193 193 361 361 361 F F F Max gen: 195, Max flush: 364 Begin loop 98 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.200696394Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.244611567Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.245303942Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.245349503Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.245366893Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.245382303Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971} {"msg":"Crucible e3668c76-bd05-4408-bc32-28a4ffa27bea has session id: 37b8d799-a4df-4473-ae67-3f3af8daeed3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.245861394Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"Upstairs opts: Upstairs UUID: e3668c76-bd05-4408-bc32-28a4ffa27bea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.245909115Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"Crucible stats registered with UUID: e3668c76-bd05-4408-bc32-28a4ffa27bea","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.245934626Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.276664692Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.276699893Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971} {"msg":"e3668c76-bd05-4408-bc32-28a4ffa27bea active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.276725454Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.276742654Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"client":"0","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.276772895Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"client":"0","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.276792085Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"client":"1","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.276809205Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"client":"1","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.276825736Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"client":"2","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.276842616Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"client":"2","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.276859156Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"":"io task","client":"2","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.276890737Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"":"io task","client":"0","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.276926768Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"":"io task","client":"1","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"ds_connection connected from Ok(127.0.0.1:55268)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.276964529Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"":"io task","client":"2","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"ds_connection connected from Ok(127.0.0.1:39466)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.27700944Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"":"io task","client":"1","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"ds_connection connected from Ok(127.0.0.1:35953)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.27705029Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"":"io task","client":"0","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.277801327Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"client":"2","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.277852178Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"client":"2","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.277893239Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"client":"1","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.277927189Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"client":"1","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.27795693Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"client":"0","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.277975301Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"client":"0","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.278409Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"client":"2","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.278525133Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"client":"1","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.278550833Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"client":"0","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.278953942Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"client":"2","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.278980592Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"client":"2","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.278994883Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.279100845Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"client":"0","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.279123655Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"client":"0","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.279137956Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.279193497Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"client":"1","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.279212968Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"client":"1","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"[0]R flush_numbers[0..12]: [355, 355, 360, 364, 364, 348, 350, 339, 362, 362, 357, 364]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.279227848Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"":"downstairs","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"[0]R generation[0..12]: [191, 191, 192, 195, 195, 186, 188, 182, 194, 194, 192, 195]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.279242278Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"":"downstairs","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.279256568Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"":"downstairs","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"[1]R flush_numbers[0..12]: [355, 355, 360, 364, 364, 348, 350, 339, 362, 362, 357, 364]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.279270389Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"":"downstairs","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"[1]R generation[0..12]: [191, 191, 192, 195, 195, 186, 188, 182, 194, 194, 192, 195]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.279284209Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"":"downstairs","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.279297989Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"":"downstairs","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"[2]R flush_numbers[0..12]: [355, 355, 360, 364, 364, 348, 350, 339, 362, 362, 357, 364]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.279311739Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"":"downstairs","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"[2]R generation[0..12]: [191, 191, 192, 195, 195, 186, 188, 182, 194, 194, 192, 195]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.2793255Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"":"downstairs","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.27933938Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"":"downstairs","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.27935258Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"":"downstairs","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.279369791Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"":"downstairs","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"Next flush: 365","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.279383871Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"":"downstairs","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"Wait for a query_work_queue command to finish before sending IO All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.279405592Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"":"downstairs","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.279419192Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.279431182Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"client":"0","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.279444942Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"client":"1","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.279458583Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"client":"2","":"downstairs","session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.279472073Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.279484363Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"e3668c76-bd05-4408-bc32-28a4ffa27bea is now active with session: 37b8d799-a4df-4473-ae67-3f3af8daeed3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.279497084Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-28T04:57:19.279509174Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971,"session_id":"37b8d799-a4df-4473-ae67-3f3af8daeed3"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:19.279524154Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 271 len 8192 data: 19 17 02/30 Flush 03/30 Write block 33 len 12288 data: 12 12 13 04/30 Write block 61 len 4096 data: 15 05/30 Read block 485 len 4096 06/30 Read block 105 len 20480 07/30 Write block 16 len 16384 data: 14 10 14 20 08/30 Read block 538 len 36864 09/30 Read block 538 len 40960 10/30 Read block 224 len 8192 11/30 Flush 12/30 Flush 13/30 Write block 583 len 8192 data: 16 16 14/30 Read block 48 len 36864 15/30 Read block 405 len 28672 16/30 Flush 17/30 Write block 120 len 32768 data: 11 15 13 17 16 15 16 14 18/30 Flush 19/30 Write block 493 len 16384 data: 11 11 13 14 20/30 Read block 532 len 8192 21/30 Read block 284 len 32768 22/30 Read block 163 len 12288 23/30 Flush 24/30 Write block 158 len 16384 data: 17 17 18 19 25/30 Read block 190 len 8192 26/30 Flush 27/30 Write block 382 len 16384 data: 14 13 12 11 28/30 Write block 565 len 24576 data: 7 10 11 10 14 18 29/30 Write block 189 len 36864 data: 18 17 17 17 16 19 20 16 19 30/30 Write block 485 len 4096 data: 18 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 4 Done Done Done false 1027 Acked Write 6 Done Sent Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 3 2 7 Done 2 1 2 5 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:57:19.297173597Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1971} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 196 196 196 366 366 366 F F F 1 020-039 196 196 196 366 366 366 F F F 2 040-059 192 192 192 360 360 360 F F F 3 060-079 196 196 196 366 366 366 F F F 4 080-099 195 195 195 364 364 364 F F F 5 100-119 186 186 186 348 348 348 F F F 6 120-139 196 196 196 369 369 369 F F F 7 140-159 196 196 196 371 371 371 F F F 8 160-179 196 196 196 371 371 371 F F F 9 180-199 194 194 194 362 362 362 T T T 10 200-219 192 192 192 357 357 357 F F F 11 220-239 195 195 195 364 364 364 F F F 12 240-259 192 192 192 360 360 360 F F F 13 260-279 196 196 196 365 365 365 F F F 14 280-299 184 184 184 342 342 342 F F F 15 300-319 192 192 192 360 360 360 F F F 16 320-339 195 195 195 364 364 364 F F F 17 340-359 194 194 194 362 362 362 F F F 18 360-379 189 189 189 351 351 351 F F F 19 380-399 190 190 190 354 354 354 T T T 20 400-419 194 194 194 362 362 362 F F F 21 420-439 192 192 192 360 360 360 F F F 22 440-459 192 192 192 358 358 358 F F F 23 460-479 190 190 190 354 354 354 F F F 24 480-499 196 196 196 370 370 370 T T T 25 500-519 195 195 195 364 364 364 F F F 26 520-539 191 191 191 355 355 355 F F F 27 540-559 194 194 194 363 363 363 F F F 28 560-579 195 195 195 364 364 364 T T T 29 580-599 196 196 196 368 368 368 F F F Max gen: 196, Max flush: 371 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 196 196 196 366 366 366 F F F 1 020-039 196 196 196 366 366 366 F F F 2 040-059 192 192 192 360 360 360 F F F 3 060-079 196 196 196 366 366 366 F F F 4 080-099 195 195 195 364 364 364 F F F 5 100-119 186 186 186 348 348 348 F F F 6 120-139 196 196 196 369 369 369 F F F 7 140-159 196 196 196 371 371 371 F F F 8 160-179 196 196 196 371 371 371 F F F 9 180-199 197 197 197 372 372 372 F F F 10 200-219 192 192 192 357 357 357 F F F 11 220-239 195 195 195 364 364 364 F F F 12 240-259 192 192 192 360 360 360 F F F 13 260-279 196 196 196 365 365 365 F F F 14 280-299 184 184 184 342 342 342 F F F 15 300-319 192 192 192 360 360 360 F F F 16 320-339 195 195 195 364 364 364 F F F 17 340-359 194 194 194 362 362 362 F F F 18 360-379 189 189 189 351 351 351 F F F 19 380-399 197 197 197 372 372 372 F F F 20 400-419 194 194 194 362 362 362 F F F 21 420-439 192 192 192 360 360 360 F F F 22 440-459 192 192 192 358 358 358 F F F 23 460-479 190 190 190 354 354 354 F F F 24 480-499 197 197 197 372 372 372 F F F 25 500-519 195 195 195 364 364 364 F F F 26 520-539 191 191 191 355 355 355 F F F 27 540-559 194 194 194 363 363 363 F F F 28 560-579 197 197 197 372 372 372 F F F 29 580-599 196 196 196 368 368 368 F F F Max gen: 197, Max flush: 372 Begin loop 99 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.558804305Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.602977893Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.603664898Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ccd99baabd70fa07cf893ca85ae393437a35a59a\",\n git_commit_timestamp: \"2024-11-28T04:33:57.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.603700799Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.603715299Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.603729909Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979} {"msg":"Crucible dc64b778-9524-42f0-a5f0-fce61056e252 has session id: c8df86f9-eb14-4a2c-ba86-39c4abf24523","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.604160869Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"Upstairs opts: Upstairs UUID: dc64b778-9524-42f0-a5f0-fce61056e252, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.60420707Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"Crucible stats registered with UUID: dc64b778-9524-42f0-a5f0-fce61056e252","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.60423181Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.63508623Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.63511604Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979} {"msg":"dc64b778-9524-42f0-a5f0-fce61056e252 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.635185932Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.635205022Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"client":"0","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.635228263Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"client":"0","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.635242903Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"client":"1","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.635257113Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"client":"1","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.635270654Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"client":"2","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.635291844Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"client":"2","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.635318195Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"":"io task","client":"2","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.635341985Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"":"io task","client":"0","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.635367506Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"":"io task","client":"1","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"ds_connection connected from Ok(127.0.0.1:51821)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.635515289Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"":"io task","client":"1","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"ds_connection connected from Ok(127.0.0.1:56390)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.635648002Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"":"io task","client":"2","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"ds_connection connected from Ok(127.0.0.1:52110)","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.635681173Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"":"io task","client":"0","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.636347897Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"client":"2","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.636387288Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"client":"2","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.636419379Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"client":"0","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.636449879Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"client":"0","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.63647413Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"client":"1","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.63648896Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"client":"1","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.637025332Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"client":"0","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.637055963Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"client":"2","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.637072933Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"client":"1","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.637589184Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"client":"1","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.637618285Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"client":"1","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.637632695Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.637645455Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"client":"2","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.637658996Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"client":"2","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.637672516Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.637684646Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"client":"0","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.637698196Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"client":"0","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"[0]R flush_numbers[0..12]: [366, 366, 360, 366, 364, 348, 369, 371, 371, 372, 357, 364]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.637712287Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"":"downstairs","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"[0]R generation[0..12]: [196, 196, 192, 196, 195, 186, 196, 196, 196, 197, 192, 195]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.637729987Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"":"downstairs","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.637744597Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"":"downstairs","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"[1]R flush_numbers[0..12]: [366, 366, 360, 366, 364, 348, 369, 371, 371, 372, 357, 364]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.637758388Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"":"downstairs","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"[1]R generation[0..12]: [196, 196, 192, 196, 195, 186, 196, 196, 196, 197, 192, 195]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.637772138Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"":"downstairs","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.637786088Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"":"downstairs","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"[2]R flush_numbers[0..12]: [366, 366, 360, 366, 364, 348, 369, 371, 371, 372, 357, 364]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.637799899Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"":"downstairs","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"[2]R generation[0..12]: [196, 196, 192, 196, 195, 186, 196, 196, 196, 197, 192, 195]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.637813639Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"":"downstairs","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.637827509Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"":"downstairs","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.63784058Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"":"downstairs","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {Wait for a query_work_queue command to finish before sending IO "msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.63786303Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"":"downstairs","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"Next flush: 373","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.63787624Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"":"downstairs","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.637889261Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"":"downstairs","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.637902571Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.637914701Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"client":"0","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.637929861Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"client":"1","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.637943282Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"client":"2","":"downstairs","session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.637956582Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.637968842Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"dc64b778-9524-42f0-a5f0-fce61056e252 is now active with session: c8df86f9-eb14-4a2c-ba86-39c4abf24523","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.637981743Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.637994623Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979,"session_id":"c8df86f9-eb14-4a2c-ba86-39c4abf24523"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-11-28T04:57:23.638017813Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 596 len 4096 data: 8 02/30 Write block 300 len 20480 data: 16 14 14 13 12 03/30 Write block 434 len 28672 data: 18 17 15 11 11 14 16 04/30 Read block 453 len 28672 05/30 Read block 86 len 8192 06/30 Read block 426 len 16384 07/30 Write block 391 len 36864 data: 14 15 16 13 14 15 12 11 13 08/30 Read block 546 len 20480 09/30 Write block 160 len 24576 data: 19 20 18 18 19 16 10/30 Read block 484 len 16384 11/30 Write block 124 len 4096 data: 17 12/30 Flush 13/30 Write block 240 len 40960 data: 15 14 13 16 14 16 18 20 21 20 14/30 Write block 129 len 28672 data: 12 15 16 17 15 15 16 15/30 Flush 16/30 Write block 180 len 24576 data: 13 11 14 18 20 20 17/30 Read block 93 len 36864 18/30 Read block 534 len 36864 19/30 Read block 553 len 24576 20/30 Read block 228 len 12288 21/30 Write block 58 len 32768 data: 18 17 16 16 13 13 13 16 22/30 Write block 103 len 4096 data: 13 23/30 Read block 324 len 4096 24/30 Write block 414 len 32768 data: 12 11 8 7 4 7 12 14 25/30 Read block 287 len 40960 26/30 Read block 46 len 8192 27/30 Read block 158 len 28672 28/30 Write block 25 len 40960 data: 19 21 20 19 17 13 13 12 13 13 29/30 Write block 396 len 4096 data: 16 30/30 Write block 569 len 28672 data: 15 19 18 18 20 23 24 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 6 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-28T04:57:23.65725225Z","hostname":"w-01JDRHKFER51NFT41MVVTWQSYN","pid":1979} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 196 196 196 366 366 366 F F F 1 020-039 196 196 196 366 366 366 T T T 2 040-059 192 192 192 360 360 360 T T T 3 060-079 196 196 196 366 366 366 T T T 4 080-099 195 195 195 364 364 364 F F F 5 100-119 186 186 186 348 348 348 T T T 6 120-139 198 198 198 374 374 374 F F F 7 140-159 196 196 196 371 371 371 F F F 8 160-179 198 198 198 373 373 373 F F F 9 180-199 197 197 197 372 372 372 T T T 10 200-219 192 192 192 357 357 357 F F F 11 220-239 195 195 195 364 364 364 F F F 12 240-259 198 198 198 374 374 374 F F F 13 260-279 196 196 196 365 365 365 F F F 14 280-299 184 184 184 342 342 342 F F F 15 300-319 198 198 198 373 373 373 F F F 16 320-339 195 195 195 364 364 364 F F F 17 340-359 194 194 194 362 362 362 F F F 18 360-379 189 189 189 351 351 351 F F F 19 380-399 198 198 198 373 373 373 T T T 20 400-419 194 194 194 362 362 362 T T T 21 420-439 198 198 198 373 373 373 T T T 22 440-459 198 198 198 373 373 373 F F F 23 460-479 190 190 190 354 354 354 F F F 24 480-499 197 197 197 372 372 372 F F F 25 500-519 195 195 195 364 364 364 F F F 26 520-539 191 191 191 355 355 355 F F F 27 540-559 194 194 194 363 363 363 F F F 28 560-579 197 197 197 372 372 372 T T T 29 580-599 198 198 198 373 373 373 F F F Max gen: 198, Max flush: 374 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 196 196 196 366 366 366 F F F 1 020-039 199 199 199 375 375 375 F F F 2 040-059 199 199 199 375 375 375 F F F 3 060-079 199 199 199 375 375 375 F F F 4 080-099 195 195 195 364 364 364 F F F 5 100-119 199 199 199 375 375 375 F F F 6 120-139 198 198 198 374 374 374 F F F 7 140-159 196 196 196 371 371 371 F F F 8 160-179 198 198 198 373 373 373 F F F 9 180-199 199 199 199 375 375 375 F F F 10 200-219 192 192 192 357 357 357 F F F 11 220-239 195 195 195 364 364 364 F F F 12 240-259 198 198 198 374 374 374 F F F 13 260-279 196 196 196 365 365 365 F F F 14 280-299 184 184 184 342 342 342 F F F 15 300-319 198 198 198 373 373 373 F F F 16 320-339 195 195 195 364 364 364 F F F 17 340-359 194 194 194 362 362 362 F F F 18 360-379 189 189 189 351 351 351 F F F 19 380-399 199 199 199 375 375 375 F F F 20 400-419 199 199 199 375 375 375 F F F 21 420-439 199 199 199 375 375 375 F F F 22 440-459 198 198 198 373 373 373 F F F 23 460-479 190 190 190 354 354 354 F F F 24 480-499 197 197 197 372 372 372 F F F 25 500-519 195 195 195 364 364 364 F F F 26 520-539 191 191 191 355 355 355 F F F 27 540-559 194 194 194 363 363 363 F F F 28 560-579 199 199 199 375 375 375 F F F 29 580-599 198 198 198 373 373 373 F F F Max gen: 199, Max flush: 375 7:19 Test duration Test completed real 7:19.806265487 user 55.195153527 sys 17.445487162 trap 0.073110496 tflt 0.031151512 dflt 0.070185556 kflt 0.005943995 lock 11:43:20.066607480 slp 35:39.382800135 lat 5.153724258 stop 0.057606735