Turn off color for downstairs dump Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.2937935 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.050417103 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.0483622 Region set with 3 regions was created /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1184 1185 1186 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.268042959Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.268577449Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.268597108Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.268612727Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188} {"msg":"Crucible 10e03898-90ee-4b51-a55e-7d303f8d1fb7 has session id: d9e269e1-cfe1-44e4-8bd7-e362548346e0","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.269102039Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"Upstairs opts: Upstairs UUID: 10e03898-90ee-4b51-a55e-7d303f8d1fb7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.269248101Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"Crucible stats registered with UUID: 10e03898-90ee-4b51-a55e-7d303f8d1fb7","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.269276309Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.306060216Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188} {"msg":"10e03898-90ee-4b51-a55e-7d303f8d1fb7 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.306104513Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.306126732Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"client":"0","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.306207718Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"client":"0","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.306235646Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"client":"1","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.306257565Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"client":"1","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.306278244Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"client":"2","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.306299122Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"client":"2","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.306320391Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.30634154Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.306362559Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:42436)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.306511341Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:42832)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.306560618Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:36417)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.306595536Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.308333067Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"client":"1","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.308360656Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"client":"1","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.308375725Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"client":"2","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.308390114Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"client":"2","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.308404253Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"client":"0","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.308425822Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"client":"0","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.309564868Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"client":"1","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.309604495Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"client":"0","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.309635504Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"client":"2","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.310617588Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"client":"1","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.310653356Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"client":"1","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.310676515Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.310695964Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"client":"2","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.310717252Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"client":"2","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.310737611Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.31075684Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"client":"0","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.310778169Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"client":"0","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.310840735Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"":"downstairs","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.310868394Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"":"downstairs","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.310884883Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"":"downstairs","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.310899292Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"":"downstairs","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.310916451Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"":"downstairs","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.31093118Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"":"downstairs","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.31094485Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"":"downstairs","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.310958509Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"":"downstairs","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.310972698Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"":"downstairs","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.310985907Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"":"downstairs","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.310999656Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"":"downstairs","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.311012595Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"":"downstairs","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.311099721Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"":"downstairs","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.31111854Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.311131209Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"client":"0","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.311144758Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"client":"1","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.311157877Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"client":"2","":"downstairs","session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.311171057Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.311183296Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"10e03898-90ee-4b51-a55e-7d303f8d1fb7 is now active with session: d9e269e1-cfe1-44e4-8bd7-e362548346e0","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.311198695Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.311210934Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188,"session_id":"d9e269e1-cfe1-44e4-8bd7-e362548346e0"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:03.311223214Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188} Wait for a query_work_queue command to finish before sending IO Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 5 Acked 1004 Read 88 Done Done Done false 6 Acked 1005 Read 256 Done Done Done false 7 Acked 1006 Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 7 6 5 4 3 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 8 7 6 5 4 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:28:07.566364567Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1188} Begin loop 1 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.673099102Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.673426693Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.673455722Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.67348131Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190} {"msg":"Crucible 8b6b737f-4077-43d1-bbf4-64023a98b9ca has session id: bb9b1628-b301-475a-ac18-e8c0c9fead57","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.673554286Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"Upstairs opts: Upstairs UUID: 8b6b737f-4077-43d1-bbf4-64023a98b9ca, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.673600523Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"Crucible stats registered with UUID: 8b6b737f-4077-43d1-bbf4-64023a98b9ca","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.673623812Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.705285409Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190} {"msg":"8b6b737f-4077-43d1-bbf4-64023a98b9ca active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.705316947Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.705335556Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"client":"0","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.705363144Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"client":"0","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.705381223Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"client":"1","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.705398293Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"client":"1","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.705414801Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"client":"2","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.705431871Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"client":"2","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.70544868Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.705466269Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"":"io task","client":"0","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.705483558Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"ds_connection connected from Ok(127.0.0.1:40675)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.705648008Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"ds_connection connected from Ok(127.0.0.1:44387)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.705749953Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"ds_connection connected from Ok(127.0.0.1:64765)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.70578988Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"":"io task","client":"0","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.706419075Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"client":"1","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.706465352Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"client":"1","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.70649783Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"client":"2","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.706516679Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"client":"2","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.706539328Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"client":"0","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.706556417Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"client":"0","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.707084867Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"client":"1","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.707113085Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"client":"0","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.707128495Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"client":"2","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.707692032Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"client":"1","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.707717891Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"client":"1","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.70773192Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.70774448Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"client":"2","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.707761299Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"client":"2","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.707775088Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.707787217Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"client":"0","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.707800737Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"client":"0","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.707814516Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"":"downstairs","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.707828005Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"":"downstairs","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.707842234Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"":"downstairs","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.707855993Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"":"downstairs","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.707869442Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"":"downstairs","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.707883412Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"":"downstairs","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.707896801Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"":"downstairs","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.70791016Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"":"downstairs","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.707924059Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"":"downstairs","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.707936809Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"":"downstairs","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,Wait for a query_work_queue command to finish before sending IO "time":"2024-09-26T14:28:07.707952208Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"":"downstairs","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.707999695Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"":"downstairs","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.708014094Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"":"downstairs","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.708027953Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.708040133Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"client":"0","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.708053702Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"client":"1","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.708067081Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"client":"2","":"downstairs","session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.708080581Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.70809286Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"8b6b737f-4077-43d1-bbf4-64023a98b9ca is now active with session: bb9b1628-b301-475a-ac18-e8c0c9fead57","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.708105679Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.708117459Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190,"session_id":"bb9b1628-b301-475a-ac18-e8c0c9fead57"} {"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:07.708136207Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 398 len 24576 data: 2 2 2 2 2 2 02/30 Read block 428 len 28672 03/30 Write block 63 len 28672 data: 2 2 2 2 2 2 2 04/30 Read block 578 len 40960 05/30 Flush 06/30 Write block 406 len 4096 data: 2 07/30 Read block 296 len 28672 08/30 Write block 239 len 36864 data: 2 2 2 2 2 2 2 2 2 09/30 Read block 101 len 40960 10/30 Flush 11/30 Flush 12/30 Write block 76 len 16384 data: 2 2 2 2 13/30 Write block 533 len 4096 data: 2 14/30 Write block 385 len 12288 data: 2 2 2 15/30 Read block 38 len 24576 16/30 Write block 395 len 20480 data: 2 2 2 3 3 17/30 Write block 191 len 32768 data: 2 2 2 2 2 2 2 2 18/30 Write block 24 len 40960 data: 2 2 2 2 2 2 2 2 2 2 19/30 Read block 284 len 12288 20/30 Read block 489 len 12288 21/30 Write block 8 len 24576 data: 2 2 2 2 2 2 22/30 Read block 119 len 24576 23/30 Write block 568 len 8192 data: 2 2 24/30 Flush 25/30 Write block 406 len 4096 data: 3 26/30 Write block 273 len 32768 data: 2 2 2 2 2 2 2 2 27/30 Read block 448 len 4096 28/30 Write block 284 len 24576 data: 2 2 2 2 2 2 29/30 Write block 293 len 32768 data: 2 2 2 2 2 2 2 2 30/30 Read block 270 len 20480 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 1 Done Done Done false 26 Acked 1025 Write 8 Done Done Done false 27 Acked 1026 Read 1 Done Done Done false 28 Acked 1027 Write 6 Done Done Done false 29 Acked 1028 Write 8 Done Sent Sent false 30 Acked 1029 Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 6 4 4 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:28:07.732312318Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1190} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 5 5 5 F F F 1 020-039 2 2 2 5 5 5 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 2 2 2 5 5 5 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 2 2 2 5 5 5 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 2 2 2 3 3 3 F F F 12 240-259 2 2 2 3 3 3 F F F 13 260-279 1 1 1 1 1 1 T T T 14 280-299 1 1 1 1 1 1 T T T 15 300-319 1 1 1 1 1 1 T T T 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 2 2 2 5 5 5 F F F 20 400-419 2 2 2 3 3 3 T T T 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 2 2 2 5 5 5 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 2 2 2 5 5 5 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 5 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 5 5 5 F F F 1 020-039 2 2 2 5 5 5 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 2 2 2 5 5 5 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 2 2 2 5 5 5 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 2 2 2 3 3 3 F F F 12 240-259 2 2 2 3 3 3 F F F 13 260-279 3 3 3 6 6 6 F F F 14 280-299 3 3 3 6 6 6 F F F 15 300-319 3 3 3 6 6 6 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 2 2 2 5 5 5 F F F 20 400-419 3 3 3 6 6 6 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 2 2 2 5 5 5 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 2 2 2 5 5 5 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 6 Begin loop 2 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.088119441Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.088447912Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.088467251Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.08848897Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198} {"msg":"Crucible b50539b5-6af7-45f7-bced-a5f00c288f4f has session id: 96690fbf-251e-453c-9560-e664992cc265","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.08865346Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"Upstairs opts: Upstairs UUID: b50539b5-6af7-45f7-bced-a5f00c288f4f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.088698958Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"Crucible stats registered with UUID: b50539b5-6af7-45f7-bced-a5f00c288f4f","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.088725206Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.119959547Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198} {"msg":"b50539b5-6af7-45f7-bced-a5f00c288f4f active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.119996045Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.120022524Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"client":"0","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.120054442Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"client":"0","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.120075861Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"client":"1","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.120107049Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"client":"1","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.120139497Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"client":"2","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.120174955Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"client":"2","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.120222963Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.12025765Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.120289749Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"ds_connection connected from Ok(127.0.0.1:36033)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.120320847Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"ds_connection connected from Ok(127.0.0.1:61625)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.120353025Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"ds_connection connected from Ok(127.0.0.1:40952)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.120472938Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.1211462Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"client":"0","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.121185758Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"client":"0","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.121216326Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"client":"2","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.121244435Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"client":"2","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.121309251Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"client":"1","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.12133446Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"client":"1","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.121892478Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"client":"0","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.121921756Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"client":"2","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.121938595Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"client":"1","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.122457216Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"client":"2","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.122480824Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"client":"2","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.122495074Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.122578569Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"client":"1","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.122611007Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"client":"1","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.122632386Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.122645305Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"client":"0","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.122658894Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"client":"0","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"[0]R flush_numbers[0..12]: [5, 5, 1, 5, 1, 1, 1, 1, 1, 5, 1, 3]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.122672704Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"":"downstairs","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"[0]R generation[0..12]: [2, 2, 1, 2, 1, 1, 1, 1, 1, 2, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.122686363Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"":"downstairs","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.122703972Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"":"downstairs","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"[1]R flush_numbers[0..12]: [5, 5, 1, 5, 1, 1, 1, 1, 1, 5, 1, 3]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.122717851Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"":"downstairs","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"[1]R generation[0..12]: [2, 2, 1, 2, 1, 1, 1, 1, 1, 2, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.122731161Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"":"downstairs","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.12274509Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"":"downstairs","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"[2]R flush_numbers[0..12]: [5, 5, 1, 5, 1, 1, 1, 1, 1, 5, 1, 3]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.122758689Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"":"downstairs","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"[2]R generation[0..12]: [2, 2, 1, 2, 1, 1, 1, 1, 1, 2, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.122772088Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"":"downstairs","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.122785937Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"":"downstairs","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-26T14:28:12.122799437Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"":"downstairs","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.122822495Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"":"downstairs","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"Next flush: 7","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.122846324Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"":"downstairs","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.122864783Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"":"downstairs","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.122887761Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.12290996Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"client":"0","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.122932089Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"client":"1","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.122946478Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"client":"2","":"downstairs","session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-09-26T14:28:12.122962947Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.122979476Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"b50539b5-6af7-45f7-bced-a5f00c288f4f is now active with session: 96690fbf-251e-453c-9560-e664992cc265","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.122992286Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.123020504Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198,"session_id":"96690fbf-251e-453c-9560-e664992cc265"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:12.123033403Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 288 len 12288 data: 3 3 2 02/30 Write block 342 len 16384 data: 2 2 2 2 03/30 Write block 263 len 32768 data: 2 2 2 2 2 2 2 2 04/30 Read block 432 len 36864 05/30 Read block 100 len 8192 06/30 Flush 07/30 Write block 449 len 36864 data: 2 2 2 2 2 2 2 2 2 08/30 Read block 513 len 8192 09/30 Read block 327 len 32768 10/30 Read block 520 len 24576 11/30 Read block 228 len 24576 12/30 Read block 524 len 16384 13/30 Write block 486 len 24576 data: 2 2 2 2 2 2 14/30 Read block 569 len 28672 15/30 Read block 130 len 4096 16/30 Flush 17/30 Write block 48 len 12288 data: 2 2 2 18/30 Write block 372 len 12288 data: 2 2 2 19/30 Write block 509 len 8192 data: 2 2 20/30 Write block 261 len 28672 data: 2 2 3 3 3 3 3 21/30 Flush 22/30 Write block 377 len 16384 data: 2 2 2 2 23/30 Flush 24/30 Write block 277 len 36864 data: 3 3 3 3 2 2 2 3 3 25/30 Read block 153 len 8192 26/30 Write block 289 len 16384 data: 4 3 2 2 27/30 Read block 3 len 8192 28/30 Write block 455 len 20480 data: 3 3 3 2 2 29/30 Read block 514 len 24576 30/30 Write block 111 len 16384 data: 2 2 2 2 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 9 Done Done Done false 25 Acked 1024 Read 2 Done Done Done false 26 Acked 1025 Write 4 Done Done Done false 27 Acked 1026 Read 2 Done Done Done false 28 Acked 1027 Write 5 Done Done Done false 29 Acked 1028 Read 6 Done Done Sent false 30 Acked 1029 Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 6 6 5 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:28:12.146720022Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1198} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 5 5 5 F F F 1 020-039 2 2 2 5 5 5 F F F 2 040-059 4 4 4 9 9 9 F F F 3 060-079 2 2 2 5 5 5 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 T T T 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 2 2 2 5 5 5 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 2 2 2 3 3 3 F F F 12 240-259 2 2 2 3 3 3 F F F 13 260-279 4 4 4 9 9 9 T T T 14 280-299 4 4 4 7 7 7 T T T 15 300-319 3 3 3 6 6 6 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 4 4 4 7 7 7 F F F 18 360-379 4 4 4 10 10 10 F F F 19 380-399 4 4 4 10 10 10 F F F 20 400-419 3 3 3 6 6 6 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 4 4 4 8 8 8 T T T 23 460-479 1 1 1 1 1 1 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 4 4 4 9 9 9 F F F 26 520-539 2 2 2 5 5 5 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 2 2 2 5 5 5 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 4, Max flush: 10 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 5 5 5 F F F 1 020-039 2 2 2 5 5 5 F F F 2 040-059 4 4 4 9 9 9 F F F 3 060-079 2 2 2 5 5 5 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 5 5 5 11 11 11 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 2 2 2 5 5 5 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 2 2 2 3 3 3 F F F 12 240-259 2 2 2 3 3 3 F F F 13 260-279 5 5 5 11 11 11 F F F 14 280-299 5 5 5 11 11 11 F F F 15 300-319 3 3 3 6 6 6 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 4 4 4 7 7 7 F F F 18 360-379 4 4 4 10 10 10 F F F 19 380-399 4 4 4 10 10 10 F F F 20 400-419 3 3 3 6 6 6 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 5 5 5 11 11 11 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 4 4 4 9 9 9 F F F 26 520-539 2 2 2 5 5 5 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 2 2 2 5 5 5 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 5, Max flush: 11 Begin loop 3 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.501689822Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.502095439Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.502134477Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.502166285Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206} {"msg":"Crucible 4bbbeb84-0486-4394-892b-8884907e16d9 has session id: ec169d5c-d225-4d2b-8af5-d61bb1dccf41","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.502192813Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"Upstairs opts: Upstairs UUID: 4bbbeb84-0486-4394-892b-8884907e16d9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.502238941Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"Crucible stats registered with UUID: 4bbbeb84-0486-4394-892b-8884907e16d9","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.502263849Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.533977193Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206} {"msg":"4bbbeb84-0486-4394-892b-8884907e16d9 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.534008091Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.53402805Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"client":"0","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.534059178Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"client":"0","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.534077008Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"client":"1","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.534094367Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"client":"1","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.534111066Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"client":"2","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.534127985Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"client":"2","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.534148773Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.534168772Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.534186141Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"ds_connection connected from Ok(127.0.0.1:62286)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.534232969Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"ds_connection connected from Ok(127.0.0.1:34774)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.534358432Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"ds_connection connected from Ok(127.0.0.1:35664)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.534515263Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.535052652Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"client":"1","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.535079291Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"client":"1","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.535189205Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"client":"2","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.53526501Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"client":"2","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.535295468Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"client":"0","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.535324557Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"client":"0","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.535661548Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"client":"1","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.535690016Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"client":"2","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.535961601Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"client":"0","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.536250574Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"client":"1","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.536287902Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"client":"1","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.536316391Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.53633824Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"client":"2","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.536361348Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"client":"2","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.536387547Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.536465542Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"client":"0","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.536491551Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"client":"0","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"[0]R flush_numbers[0..12]: [5, 5, 9, 5, 1, 11, 1, 1, 1, 5, 1, 3]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.53650672Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"":"downstairs","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"[0]R generation[0..12]: [2, 2, 4, 2, 1, 5, 1, 1, 1, 2, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.536528509Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"":"downstairs","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.536543698Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"":"downstairs","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"[1]R flush_numbers[0..12]: [5, 5, 9, 5, 1, 11, 1, 1, 1, 5, 1, 3]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.536557427Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"":"downstairs","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"[1]R generation[0..12]: [2, 2, 4, 2, 1, 5, 1, 1, 1, 2, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.536577516Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"":"downstairs","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.536592815Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"":"downstairs","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"[2]R flush_numbers[0..12]: [5, 5, 9, 5, 1, 11, 1, 1, 1, 5, 1, 3]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.536606424Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"":"downstairs","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"[2]R generation[0..12]: [2, 2, 4, 2, 1, 5, 1, 1, 1, 2, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.536630043Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"":"downstairs","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.536645392Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"":"downstairs","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.536660821Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"":"downstairs","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.53667866Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"":"downstairs","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 12","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.536698019Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"":"downstairs","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.536711638Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"":"downstairs","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.536725067Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.536737297Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"client":"0","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.536750616Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"client":"1","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.536763645Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"client":"2","":"downstairs","session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.536776764Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.536789004Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"4bbbeb84-0486-4394-892b-8884907e16d9 is now active with session: ec169d5c-d225-4d2b-8af5-d61bb1dccf41","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.536801793Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.536813673Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206,"session_id":"ec169d5c-d225-4d2b-8af5-d61bb1dccf41"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:16.536825802Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 184 len 8192 data: 2 2 02/30 Read block 187 len 20480 03/30 Write block 230 len 4096 data: 2 04/30 Read block 63 len 32768 05/30 Read block 428 len 36864 06/30 Flush 07/30 Write block 583 len 4096 data: 2 08/30 Write block 10 len 36864 data: 3 3 3 3 2 2 2 2 2 09/30 Read block 534 len 36864 10/30 Read block 542 len 32768 11/30 Read block 72 len 28672 12/30 Read block 232 len 24576 13/30 Read block 396 len 24576 14/30 Write block 59 len 12288 data: 2 2 2 15/30 Read block 159 len 4096 16/30 Write block 369 len 20480 data: 2 2 2 3 3 17/30 Read block 455 len 36864 18/30 Write block 164 len 20480 data: 2 2 2 2 2 19/30 Read block 288 len 40960 20/30 Read block 540 len 40960 21/30 Write block 382 len 8192 data: 2 2 22/30 Read block 212 len 4096 23/30 Read block 514 len 32768 24/30 Read block 29 len 40960 25/30 Write block 514 len 28672 data: 2 2 2 2 2 2 2 26/30 Read block 431 len 28672 27/30 Write block 57 len 28672 data: 2 2 3 3 3 2 3 28/30 Read block 28 len 24576 29/30 Read block 143 len 4096 30/30 Read block 561 len 20480 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 7 Acked 1006 Write 1 Done Done Done false 8 Acked 1007 Write 9 Done Done Done false 9 Acked 1008 Read 9 Done Done Done false 10 Acked 1009 Read 8 Done Done Done false 11 Acked 1010 Read 7 Done Done Done false 12 Acked 1011 Read 6 Done Done Done false 13 Acked 1012 Read 6 Done Done Done false 14 Acked 1013 Write 3 Done Done Done false 15 Acked 1014 Read 1 Done Done Done false 16 Acked 1015 Write 5 Done Done Done false 17 Acked 1016 Read 9 Done Done Done false 18 Acked 1017 Write 5 Done Done Done false 19 Acked 1018 Read 10 Done Done Done false 20 Acked 1019 Read 10 Done Done Done false 21 Acked 1020 Write 2 Done Done Done false 22 Acked 1021 Read 1 Done Done Done false 23 Acked 1022 Read 8 Done Done Done false 24 Acked 1023 Read 10 Done Done Done false 25 Acked 1024 Write 7 Done Done Done false 26 Acked 1025 Read 7 Done Done Done false 27 Acked 1026 Write 7 Done Done Sent false 28 Acked 1027 Read 6 Done Done Sent false 29 Acked 1028 Read 1 Done Done Sent false 30 Acked 1029 Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 4 5 Done 24 23 20 67 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1005 1005 1005 Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:28:16.558091788Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1206} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 5 5 5 T T T 1 020-039 2 2 2 5 5 5 F F F 2 040-059 4 4 4 9 9 9 T T T 3 060-079 2 2 2 5 5 5 T T T 4 080-099 1 1 1 1 1 1 F F F 5 100-119 5 5 5 11 11 11 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 T T T 9 180-199 6 6 6 12 12 12 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 6 6 6 12 12 12 F F F 12 240-259 2 2 2 3 3 3 F F F 13 260-279 5 5 5 11 11 11 F F F 14 280-299 5 5 5 11 11 11 F F F 15 300-319 3 3 3 6 6 6 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 4 4 4 7 7 7 F F F 18 360-379 4 4 4 10 10 10 T T T 19 380-399 4 4 4 10 10 10 T T T 20 400-419 3 3 3 6 6 6 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 5 5 5 11 11 11 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 4 4 4 9 9 9 T T T 26 520-539 2 2 2 5 5 5 T T T 27 540-559 1 1 1 1 1 1 F F F 28 560-579 2 2 2 5 5 5 F F F 29 580-599 1 1 1 1 1 1 T T T Max gen: 6, Max flush: 12 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 13 13 13 F F F 1 020-039 2 2 2 5 5 5 F F F 2 040-059 7 7 7 13 13 13 F F F 3 060-079 7 7 7 13 13 13 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 5 5 5 11 11 11 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 7 7 7 13 13 13 F F F 9 180-199 6 6 6 12 12 12 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 6 6 6 12 12 12 F F F 12 240-259 2 2 2 3 3 3 F F F 13 260-279 5 5 5 11 11 11 F F F 14 280-299 5 5 5 11 11 11 F F F 15 300-319 3 3 3 6 6 6 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 4 4 4 7 7 7 F F F 18 360-379 7 7 7 13 13 13 F F F 19 380-399 7 7 7 13 13 13 F F F 20 400-419 3 3 3 6 6 6 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 5 5 5 11 11 11 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 7 7 7 13 13 13 F F F 26 520-539 7 7 7 13 13 13 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 2 2 2 5 5 5 F F F 29 580-599 7 7 7 13 13 13 F F F Max gen: 7, Max flush: 13 Begin loop 4 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:20.989323758Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:20.98962994Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:20.989672648Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:20.989705226Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214} {"msg":"Crucible 477c972a-c20d-4de1-bb59-e62935590c11 has session id: dba0ea04-044f-4f99-8808-2e3b68e759d8","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:20.989730615Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"Upstairs opts: Upstairs UUID: 477c972a-c20d-4de1-bb59-e62935590c11, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:20.989778122Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"Crucible stats registered with UUID: 477c972a-c20d-4de1-bb59-e62935590c11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:20.9898058Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.021906302Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214} {"msg":"477c972a-c20d-4de1-bb59-e62935590c11 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.021934771Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.02195199Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"client":"0","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.021979648Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"client":"0","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.021997317Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"client":"1","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.022014416Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"client":"1","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.022031215Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"client":"2","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.022047954Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"client":"2","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.022064523Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.022081602Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"":"io task","client":"0","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.022101981Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"":"io task","client":"1","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:65181)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.02211975Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:37344)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.022364697Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"":"io task","client":"0","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:32859)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.022412664Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"":"io task","client":"1","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.022896616Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"client":"2","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.022925345Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"client":"2","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.023057097Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"client":"1","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.023080326Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"client":"1","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.023226948Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"client":"0","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.023264326Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"client":"0","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.023617526Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"client":"1","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.02372444Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"client":"2","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.02389671Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"client":"0","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.0242364Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"client":"2","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.024263819Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"client":"2","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.024281858Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.024308186Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"client":"1","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.024322115Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"client":"1","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.024335305Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.024404301Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"client":"0","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.02442239Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"client":"0","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"[0]R flush_numbers[0..12]: [13, 5, 13, 13, 1, 11, 1, 1, 13, 12, 1, 12]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.024436439Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"":"downstairs","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"[0]R generation[0..12]: [7, 2, 7, 7, 1, 5, 1, 1, 7, 6, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.024450358Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"":"downstairs","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.024464467Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"":"downstairs","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"[1]R flush_numbers[0..12]: [13, 5, 13, 13, 1, 11, 1, 1, 13, 12, 1, 12]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.024478247Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"":"downstairs","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"[1]R generation[0..12]: [7, 2, 7, 7, 1, 5, 1, 1, 7, 6, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.024491776Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"":"downstairs","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.024505845Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"":"downstairs","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"[2]R flush_numbers[0..12]: [13, 5, 13, 13, 1, 11, 1, 1, 13, 12, 1, 12]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.024519255Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"":"downstairs","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"[2]R generation[0..12]: [7, 2, 7, 7, 1, 5, 1, 1, 7, 6, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.024532714Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"":"downstairs","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.024546643Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"":"downstairs","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.024562562Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"":"downstairs","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.024576111Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"":"downstairs","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"Next flush: 14","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.02458886Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"":"downstairs","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.02460175Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"":"downstairs","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-09-26T14:28:21.024614879Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.024630488Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"client":"0","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.024643957Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"client":"1","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.024656857Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"client":"2","":"downstairs","session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.024670296Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.024682555Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"477c972a-c20d-4de1-bb59-e62935590c11 is now active with session: dba0ea04-044f-4f99-8808-2e3b68e759d8","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.024695325Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.024707304Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214,"session_id":"dba0ea04-044f-4f99-8808-2e3b68e759d8"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:21.024719403Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 158 len 8192 data: 2 2 02/30 Write block 528 len 28672 data: 2 2 2 2 2 3 2 03/30 Read block 523 len 32768 04/30 Read block 179 len 8192 05/30 Write block 544 len 12288 data: 2 2 2 06/30 Read block 332 len 4096 07/30 Flush 08/30 Write block 118 len 32768 data: 2 2 2 2 2 2 2 2 09/30 Write block 66 len 12288 data: 3 3 3 10/30 Write block 211 len 36864 data: 2 2 2 2 2 2 2 2 2 11/30 Read block 137 len 24576 12/30 Read block 109 len 4096 13/30 Write block 496 len 28672 data: 2 2 2 2 2 2 2 14/30 Read block 319 len 4096 15/30 Read block 380 len 16384 16/30 Write block 534 len 12288 data: 3 2 2 17/30 Read block 121 len 36864 18/30 Write block 268 len 4096 data: 3 19/30 Write block 182 len 36864 data: 2 2 3 3 2 2 2 2 2 20/30 Read block 550 len 4096 21/30 Write block 493 len 12288 data: 2 2 2 22/30 Read block 550 len 12288 23/30 Write block 517 len 8192 data: 3 3 24/30 Read block 292 len 16384 25/30 Flush 26/30 Flush 27/30 Write block 118 len 36864 data: 3 3 3 3 3 3 3 3 2 28/30 Read block 497 len 12288 29/30 Write block 188 len 28672 data: 3 3 3 3 3 3 3 30/30 Write block 333 len 12288 data: 2 2 2 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 9 Done Done Done false 28 Acked 1027 Read 3 Done Done Done false 29 Acked 1028 Write 7 Sent Sent Sent false 30 Acked 1029 Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:28:21.04614532Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1214} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 13 13 13 F F F 1 020-039 2 2 2 5 5 5 F F F 2 040-059 7 7 7 13 13 13 F F F 3 060-079 8 8 8 15 15 15 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 8 8 8 15 15 15 T T T 6 120-139 8 8 8 15 15 15 T T T 7 140-159 8 8 8 14 14 14 F F F 8 160-179 7 7 7 13 13 13 F F F 9 180-199 8 8 8 15 15 15 T T T 10 200-219 8 8 8 15 15 15 F F F 11 220-239 6 6 6 12 12 12 F F F 12 240-259 2 2 2 3 3 3 F F F 13 260-279 8 8 8 15 15 15 F F F 14 280-299 5 5 5 11 11 11 F F F 15 300-319 3 3 3 6 6 6 F F F 16 320-339 1 1 1 1 1 1 T T T 17 340-359 4 4 4 7 7 7 F F F 18 360-379 7 7 7 13 13 13 F F F 19 380-399 7 7 7 13 13 13 F F F 20 400-419 3 3 3 6 6 6 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 5 5 5 11 11 11 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 8 8 8 15 15 15 F F F 25 500-519 8 8 8 15 15 15 F F F 26 520-539 8 8 8 15 15 15 F F F 27 540-559 8 8 8 14 14 14 F F F 28 560-579 2 2 2 5 5 5 F F F 29 580-599 7 7 7 13 13 13 F F F Max gen: 8, Max flush: 15 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 13 13 13 F F F 1 020-039 2 2 2 5 5 5 F F F 2 040-059 7 7 7 13 13 13 F F F 3 060-079 8 8 8 15 15 15 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 9 9 9 16 16 16 F F F 6 120-139 9 9 9 16 16 16 F F F 7 140-159 8 8 8 14 14 14 F F F 8 160-179 7 7 7 13 13 13 F F F 9 180-199 9 9 9 16 16 16 F F F 10 200-219 8 8 8 15 15 15 F F F 11 220-239 6 6 6 12 12 12 F F F 12 240-259 2 2 2 3 3 3 F F F 13 260-279 8 8 8 15 15 15 F F F 14 280-299 5 5 5 11 11 11 F F F 15 300-319 3 3 3 6 6 6 F F F 16 320-339 9 9 9 16 16 16 F F F 17 340-359 4 4 4 7 7 7 F F F 18 360-379 7 7 7 13 13 13 F F F 19 380-399 7 7 7 13 13 13 F F F 20 400-419 3 3 3 6 6 6 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 5 5 5 11 11 11 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 8 8 8 15 15 15 F F F 25 500-519 8 8 8 15 15 15 F F F 26 520-539 8 8 8 15 15 15 F F F 27 540-559 8 8 8 14 14 14 F F F 28 560-579 2 2 2 5 5 5 F F F 29 580-599 7 7 7 13 13 13 F F F Max gen: 9, Max flush: 16 Begin loop 5 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.400613387Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.400876032Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.4009064Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.400965487Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222} {"msg":"Crucible 65772d5f-d2bd-44d6-995f-b3f593841213 has session id: 48988a71-6235-4f4f-a2f8-369daaca02b3","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.401062451Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"Upstairs opts: Upstairs UUID: 65772d5f-d2bd-44d6-995f-b3f593841213, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.401106979Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"Crucible stats registered with UUID: 65772d5f-d2bd-44d6-995f-b3f593841213","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.401129688Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.432216417Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222} {"msg":"65772d5f-d2bd-44d6-995f-b3f593841213 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.432257054Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.432284283Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"client":"0","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.432329801Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"client":"0","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.432366938Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"client":"1","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.432402066Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"client":"1","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.432435895Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"client":"2","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.432469402Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"client":"2","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.432502001Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.432535029Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.432566817Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"ds_connection connected from Ok(127.0.0.1:49509)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.432597085Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"ds_connection connected from Ok(127.0.0.1:47126)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.432715719Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"ds_connection connected from Ok(127.0.0.1:35274)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.432747577Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.433386361Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"client":"2","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.433414069Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"client":"2","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.433429098Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"client":"0","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.433448627Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"client":"0","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.433475396Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"client":"1","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.433501554Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"client":"1","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.433974487Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"client":"2","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.434071742Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"client":"1","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.434171746Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"client":"0","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.434624171Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"client":"2","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.434651419Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"client":"2","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.434666858Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.434731134Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"client":"1","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.434752043Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"client":"1","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.434766003Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.434778952Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"client":"0","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.434792251Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"client":"0","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"[0]R flush_numbers[0..12]: [13, 5, 13, 15, 1, 16, 16, 14, 13, 16, 15, 12]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.43480661Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"":"downstairs","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"[0]R generation[0..12]: [7, 2, 7, 8, 1, 9, 9, 8, 7, 9, 8, 6]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.434824399Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"":"downstairs","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.434838908Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"":"downstairs","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"[1]R flush_numbers[0..12]: [13, 5, 13, 15, 1, 16, 16, 14, 13, 16, 15, 12]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.434852667Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"":"downstairs","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"[1]R generation[0..12]: [7, 2, 7, 8, 1, 9, 9, 8, 7, 9, 8, 6]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.434866117Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"":"downstairs","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.434880016Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"":"downstairs","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"[2]R flush_numbers[0..12]: [13, 5, 13, 15, 1, 16, 16, 14, 13, 16, 15, 12]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.434893545Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"":"downstairs","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"[2]R generation[0..12]: [7, 2, 7, 8, 1, 9, 9, 8, 7, 9, 8, 6]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.434906784Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"":"downstairs","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.434920504Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"":"downstairs","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.434933483Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"":"downstairs","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.434946622Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"":"downstairs","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"Next flush: 17","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.434962971Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"":"downstairs","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.434975911Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"":"downstairs","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.43498885Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.435000689Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"client":"0","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.435013829Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"client":"1","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.435028697Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"client":"2","":"downstairs","session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.435041917Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.435053996Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"65772d5f-d2bd-44d6-995f-b3f593841213 is now active with session: 48988a71-6235-4f4f-a2f8-369daaca02b3","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.435066775Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.435078765Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222,"session_id":"48988a71-6235-4f4f-a2f8-369daaca02b3"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:25.435090924Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 159 len 28672 data: 3 2 2 2 2 3 3 02/30 Read block 56 len 12288 03/30 Flush 04/30 Write block 536 len 24576 data: 3 2 2 2 2 2 05/30 Write block 86 len 24576 data: 2 2 2 2 2 2 06/30 Write block 18 len 28672 data: 3 2 2 2 2 2 3 07/30 Read block 434 len 16384 08/30 Read block 514 len 8192 09/30 Write block 394 len 32768 data: 2 3 3 3 4 4 3 3 10/30 Read block 566 len 4096 11/30 Read block 184 len 40960 12/30 Read block 374 len 40960 13/30 Read block 115 len 8192 14/30 Read block 322 len 24576 15/30 Read block 256 len 8192 16/30 Write block 55 len 40960 data: 2 2 3 3 4 4 4 3 4 3 17/30 Write block 30 len 40960 data: 3 3 3 3 2 2 2 2 2 2 18/30 Write block 375 len 4096 data: 2 19/30 Flush 20/30 Write block 0 len 8192 data: 2 2 21/30 Read block 193 len 4096 22/30 Read block 389 len 24576 23/30 Write block 57 len 12288 data: 4 4 5 24/30 Write block 351 len 20480 data: 2 2 2 2 2 25/30 Flush 26/30 Write block 199 len 36864 data: 2 2 2 2 2 2 2 2 2 27/30 Write block 551 len 40960 data: 2 2 2 2 2 2 2 2 2 2 28/30 Read block 331 len 24576 29/30 Read block 79 len 28672 30/30 Write block 446 len 4096 data: 2 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 9 Done Done Done false 27 Acked 1026 Write 10 Done Done Done false 28 Acked 1027 Read 6 Done Done Done false 29 Acked 1028 Read 7 Done Done Done false 30 Acked 1029 Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:28:25.457639757Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1222} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 19 19 19 F F F 1 020-039 10 10 10 18 18 18 F F F 2 040-059 10 10 10 19 19 19 F F F 3 060-079 10 10 10 18 18 18 F F F 4 080-099 10 10 10 18 18 18 F F F 5 100-119 9 9 9 16 16 16 F F F 6 120-139 9 9 9 16 16 16 F F F 7 140-159 10 10 10 17 17 17 F F F 8 160-179 10 10 10 17 17 17 F F F 9 180-199 9 9 9 16 16 16 T T T 10 200-219 8 8 8 15 15 15 T T T 11 220-239 6 6 6 12 12 12 F F F 12 240-259 2 2 2 3 3 3 F F F 13 260-279 8 8 8 15 15 15 F F F 14 280-299 5 5 5 11 11 11 F F F 15 300-319 3 3 3 6 6 6 F F F 16 320-339 9 9 9 16 16 16 F F F 17 340-359 10 10 10 19 19 19 F F F 18 360-379 10 10 10 18 18 18 F F F 19 380-399 10 10 10 18 18 18 F F F 20 400-419 10 10 10 18 18 18 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 5 5 5 11 11 11 T T T 23 460-479 1 1 1 1 1 1 F F F 24 480-499 8 8 8 15 15 15 F F F 25 500-519 8 8 8 15 15 15 F F F 26 520-539 10 10 10 18 18 18 F F F 27 540-559 10 10 10 18 18 18 T T T 28 560-579 2 2 2 5 5 5 T T T 29 580-599 7 7 7 13 13 13 F F F Max gen: 10, Max flush: 19 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 19 19 19 F F F 1 020-039 10 10 10 18 18 18 F F F 2 040-059 10 10 10 19 19 19 F F F 3 060-079 10 10 10 18 18 18 F F F 4 080-099 10 10 10 18 18 18 F F F 5 100-119 9 9 9 16 16 16 F F F 6 120-139 9 9 9 16 16 16 F F F 7 140-159 10 10 10 17 17 17 F F F 8 160-179 10 10 10 17 17 17 F F F 9 180-199 11 11 11 20 20 20 F F F 10 200-219 11 11 11 20 20 20 F F F 11 220-239 6 6 6 12 12 12 F F F 12 240-259 2 2 2 3 3 3 F F F 13 260-279 8 8 8 15 15 15 F F F 14 280-299 5 5 5 11 11 11 F F F 15 300-319 3 3 3 6 6 6 F F F 16 320-339 9 9 9 16 16 16 F F F 17 340-359 10 10 10 19 19 19 F F F 18 360-379 10 10 10 18 18 18 F F F 19 380-399 10 10 10 18 18 18 F F F 20 400-419 10 10 10 18 18 18 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 11 11 11 20 20 20 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 8 8 8 15 15 15 F F F 25 500-519 8 8 8 15 15 15 F F F 26 520-539 10 10 10 18 18 18 F F F 27 540-559 11 11 11 20 20 20 F F F 28 560-579 11 11 11 20 20 20 F F F 29 580-599 7 7 7 13 13 13 F F F Max gen: 11, Max flush: 20 Begin loop 6 Downstairs to restart: 2 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.788425265Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.788714199Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.788734238Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.788749917Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230} {"msg":"Crucible 9101e155-aec0-44ef-b446-027633f82a18 has session id: 424ca558-71ff-4df6-b36e-84faffecede2","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.788897318Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"Upstairs opts: Upstairs UUID: 9101e155-aec0-44ef-b446-027633f82a18, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.788948026Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"Crucible stats registered with UUID: 9101e155-aec0-44ef-b446-027633f82a18","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.788966265Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.820491329Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230} {"msg":"9101e155-aec0-44ef-b446-027633f82a18 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.820522777Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.820546466Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"client":"0","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.820581334Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"client":"0","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.820607673Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"client":"1","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.820631591Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"client":"1","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.82065541Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"client":"2","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.820687728Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"client":"2","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.820719376Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.820749745Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.820778143Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"ds_connection connected from Ok(127.0.0.1:57151)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.820804882Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"ds_connection connected from Ok(127.0.0.1:50919)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.82082833Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"ds_connection connected from Ok(127.0.0.1:48565)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.821017719Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.821559539Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"client":"0","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.821587537Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"client":"0","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.821603106Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"client":"2","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.821617076Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"client":"2","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.821642054Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"client":"1","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.821656623Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"client":"1","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.82223824Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"client":"1","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.822266699Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"client":"2","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.822282998Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"client":"0","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.822806388Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"client":"1","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.822834267Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"client":"1","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.822850735Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.822863725Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"client":"0","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.822877294Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"client":"0","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.822890643Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.822902983Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"client":"2","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.822916302Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"client":"2","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"[0]R flush_numbers[0..12]: [19, 18, 19, 18, 18, 16, 16, 17, 17, 20, 20, 12]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.822930091Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"":"downstairs","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"[0]R generation[0..12]: [10, 10, 10, 10, 10, 9, 9, 10, 10, 11, 11, 6]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.82294387Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"":"downstairs","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.82295856Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"":"downstairs","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"[1]R flush_numbers[0..12]: [19, 18, 19, 18, 18, 16, 16, 17, 17, 20, 20, 12]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.822972609Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"":"downstairs","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"[1]R generation[0..12]: [10, 10, 10, 10, 10, 9, 9, 10, 10, 11, 11, 6]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.822986298Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"":"downstairs","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.823001147Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"":"downstairs","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"[2]R flush_numbers[0..12]: [19, 18, 19, 18, 18, 16, 16, 17, 17, 20, 20, 12]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.823015016Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"":"downstairs","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"[2]R generation[0..12]: [10, 10, 10, 10, 10, 9, 9, 10, 10, 11, 11, 6]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.823032275Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"":"downstairs","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.823046764Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"":"downstairs","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.823059794Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"":"downstairs","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.823072933Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"":"downstairs","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"Next flush: 21","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.823085612Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"":"downstairs","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.823098532Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"":"downstairs","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.823111681Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3"Wait for a query_work_queue command to finish before sending IO ,"pid":1230,"session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.82312828Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"client":"0","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.823141849Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"client":"1","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.823155048Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"client":"2","":"downstairs","session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.823168277Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.823180557Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"9101e155-aec0-44ef-b446-027633f82a18 is now active with session: 424ca558-71ff-4df6-b36e-84faffecede2","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.823193466Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.823205406Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230,"session_id":"424ca558-71ff-4df6-b36e-84faffecede2"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:29.823217605Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 360 len 12288 data: 2 2 2 02/30 Read block 246 len 24576 03/30 Read block 86 len 32768 04/30 Read block 500 len 20480 05/30 Read block 391 len 12288 06/30 Read block 320 len 40960 07/30 Read block 568 len 32768 08/30 Read block 373 len 20480 09/30 Read block 315 len 40960 10/30 Write block 521 len 36864 data: 2 2 2 2 2 2 2 3 3 11/30 Read block 547 len 28672 12/30 Read block 525 len 8192 13/30 Write block 170 len 28672 data: 2 2 2 2 2 2 2 14/30 Write block 200 len 12288 data: 3 3 3 15/30 Read block 103 len 8192 16/30 Read block 217 len 20480 17/30 Read block 63 len 8192 18/30 Write block 207 len 16384 data: 3 2 2 2 19/30 Write block 22 len 28672 data: 3 3 4 3 3 3 3 20/30 Read block 134 len 20480 21/30 Read block 232 len 12288 22/30 Read block 328 len 20480 23/30 Write block 317 len 36864 data: 2 2 2 2 2 2 2 2 2 24/30 Write block 432 len 40960 data: 2 2 2 2 2 2 2 2 2 2 25/30 Read block 381 len 28672 26/30 Write block 511 len 16384 data: 2 2 2 3 27/30 Write block 427 len 28672 data: 2 2 2 2 2 3 3 28/30 Write block 568 len 28672 data: 3 3 2 2 2 2 2 29/30 Read block 177 len 4096 30/30 Write block 258 len 20480 data: 2 2 2 3 3 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Write 3 Done Done Done false 2 Acked 1001 Read 6 Done Done Done false 3 Acked 1002 Read 8 Done Done Done false 4 Acked 1003 Read 5 Done Done Done false 5 Acked 1004 Read 3 Done Done Done false 6 Acked 1005 Read 10 Done Done Done false 7 Acked 1006 Read 8 Done Done Done false 8 Acked 1007 Read 5 Done Done Done false 9 Acked 1008 Read 10 Done Done Done false 10 Acked 1009 Write 9 Done Done Done false 11 Acked 1010 Read 7 Done Done Done false 12 Acked 1011 Read 2 Done Done Done false 13 Acked 1012 Write 7 Done Done Done false 14 Acked 1013 Write 3 Done Done Done false 15 Acked 1014 Read 2 Done Done Done false 16 Acked 1015 Read 5 Done Done Done false 17 Acked 1016 Read 2 Done Done Done false 18 Acked 1017 Write 4 Done Done Done false 19 Acked 1018 Write 7 Done Done Done false 20 Acked 1019 Read 5 Done Done Done false 21 Acked 1020 Read 3 Done Done Done false 22 Acked 1021 Read 5 Done Done Done false 23 Acked 1022 Write 9 Done Done Done false 24 Acked 1023 Write 10 Done Done Done false 25 Acked 1024 Read 7 Done Done Done false 26 Acked 1025 Write 4 Done Done Done false 27 Acked 1026 Write 7 Done Done Done false 28 Acked 1027 Write 7 Done Done Done false 29 Acked 1028 Read 1 Done Done Done false 30 Acked 1029 Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:28:29.843276089Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1230} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 19 19 19 F F F 1 020-039 10 10 10 18 18 18 T T T 2 040-059 10 10 10 19 19 19 F F F 3 060-079 10 10 10 18 18 18 F F F 4 080-099 10 10 10 18 18 18 F F F 5 100-119 9 9 9 16 16 16 F F F 6 120-139 9 9 9 16 16 16 F F F 7 140-159 10 10 10 17 17 17 F F F 8 160-179 10 10 10 17 17 17 T T T 9 180-199 11 11 11 20 20 20 F F F 10 200-219 11 11 11 20 20 20 T T T 11 220-239 6 6 6 12 12 12 F F F 12 240-259 2 2 2 3 3 3 T T T 13 260-279 8 8 8 15 15 15 T T T 14 280-299 5 5 5 11 11 11 F F F 15 300-319 3 3 3 6 6 6 T T T 16 320-339 9 9 9 16 16 16 T T T 17 340-359 10 10 10 19 19 19 F F F 18 360-379 10 10 10 18 18 18 T T T 19 380-399 10 10 10 18 18 18 F F F 20 400-419 10 10 10 18 18 18 F F F 21 420-439 1 1 1 1 1 1 T T T 22 440-459 11 11 11 20 20 20 T T T 23 460-479 1 1 1 1 1 1 F F F 24 480-499 8 8 8 15 15 15 F F F 25 500-519 8 8 8 15 15 15 T T T 26 520-539 10 10 10 18 18 18 T T T 27 540-559 11 11 11 20 20 20 F F F 28 560-579 11 11 11 20 20 20 T T T 29 580-599 7 7 7 13 13 13 F F F Max gen: 11, Max flush: 20 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 19 19 19 F F F 1 020-039 12 12 12 21 21 21 F F F 2 040-059 10 10 10 19 19 19 F F F 3 060-079 10 10 10 18 18 18 F F F 4 080-099 10 10 10 18 18 18 F F F 5 100-119 9 9 9 16 16 16 F F F 6 120-139 9 9 9 16 16 16 F F F 7 140-159 10 10 10 17 17 17 F F F 8 160-179 12 12 12 21 21 21 F F F 9 180-199 11 11 11 20 20 20 F F F 10 200-219 12 12 12 21 21 21 F F F 11 220-239 6 6 6 12 12 12 F F F 12 240-259 12 12 12 21 21 21 F F F 13 260-279 12 12 12 21 21 21 F F F 14 280-299 5 5 5 11 11 11 F F F 15 300-319 12 12 12 21 21 21 F F F 16 320-339 12 12 12 21 21 21 F F F 17 340-359 10 10 10 19 19 19 F F F 18 360-379 12 12 12 21 21 21 F F F 19 380-399 10 10 10 18 18 18 F F F 20 400-419 10 10 10 18 18 18 F F F 21 420-439 12 12 12 21 21 21 F F F 22 440-459 12 12 12 21 21 21 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 8 8 8 15 15 15 F F F 25 500-519 12 12 12 21 21 21 F F F 26 520-539 12 12 12 21 21 21 F F F 27 540-559 11 11 11 20 20 20 F F F 28 560-579 12 12 12 21 21 21 F F F 29 580-599 7 7 7 13 13 13 F F F Max gen: 12, Max flush: 21 Begin loop 7 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.291842258Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.292177679Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.292211097Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.292243765Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238} {"msg":"Crucible b1e63ff4-b0b0-40aa-9322-e706cd31a415 has session id: f26fb16e-113b-4ca0-9954-d223708ee9ae","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.29232867Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"Upstairs opts: Upstairs UUID: b1e63ff4-b0b0-40aa-9322-e706cd31a415, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.292382207Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"Crucible stats registered with UUID: b1e63ff4-b0b0-40aa-9322-e706cd31a415","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.292404276Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.323977698Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238} {"msg":"b1e63ff4-b0b0-40aa-9322-e706cd31a415 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.324009876Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.324028675Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"client":"0","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.324056393Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"client":"0","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.324074102Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"client":"1","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.324090771Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"client":"1","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.32410695Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"client":"2","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.32412406Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"client":"2","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.324145858Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.324163637Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.324191946Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"ds_connection connected from Ok(127.0.0.1:53657)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.324225454Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"ds_connection connected from Ok(127.0.0.1:38144)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.324258222Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"ds_connection connected from Ok(127.0.0.1:57680)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.324483629Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.324983261Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"client":"1","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.325025609Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"client":"1","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.325042308Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"client":"2","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.325056257Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"client":"2","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.325111734Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"client":"0","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.325127443Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"client":"0","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.325616425Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"client":"2","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.325781266Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"client":"1","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.325820704Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"client":"0","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.326141315Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"client":"2","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.326166804Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"client":"2","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.326181133Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.326370872Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"client":"1","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.326396841Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"client":"1","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.32641097Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.326438549Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"client":"0","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.326452798Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"client":"0","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"[0]R flush_numbers[0..12]: [19, 21, 19, 18, 18, 16, 16, 17, 21, 20, 21, 12]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.326466687Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"":"downstairs","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"[0]R generation[0..12]: [10, 12, 10, 10, 10, 9, 9, 10, 12, 11, 12, 6]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.326480326Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"":"downstairs","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.326494395Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"":"downstairs","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"[1]R flush_numbers[0..12]: [19, 21, 19, 18, 18, 16, 16, 17, 21, 20, 21, 12]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.326508054Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"":"downstairs","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"[1]R generation[0..12]: [10, 12, 10, 10, 10, 9, 9, 10, 12, 11, 12, 6]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.326521704Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"":"downstairs","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.326535823Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"":"downstairs","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"[2]R flush_numbers[0..12]: [19, 21, 19, 18, 18, 16, 16, 17, 21, 20, 21, 12]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.326553172Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"":"downstairs","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"[2]R generation[0..12]: [10, 12, 10, 10, 10, 9, 9, 10, 12, 11, 12, 6]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.326567061Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"":"downstairs","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.32658101Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"":"downstairs","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"Max found gen is 13","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.32659405Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"":"downstairs","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"Generation requested: 14 >= found:13","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.326607329Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"":"downstairs","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 22","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.326626338Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"":"downstairs","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.326639987Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"":"downstairs","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.326653266Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.326665146Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"client":"0","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.326678315Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"client":"1","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.326691424Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"client":"2","":"downstairs","session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.326704623Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.326725742Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"b1e63ff4-b0b0-40aa-9322-e706cd31a415 is now active with session: f26fb16e-113b-4ca0-9954-d223708ee9ae","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.326749621Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.32676336Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238,"session_id":"f26fb16e-113b-4ca0-9954-d223708ee9ae"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:34.326783139Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 503 len 16384 data: 2 2 2 2 02/30 Flush 03/30 Write block 178 len 32768 data: 2 2 2 2 3 3 4 4 04/30 Flush 05/30 Write block 124 len 8192 data: 4 4 06/30 Write block 35 len 36864 data: 3 3 3 3 3 2 2 2 2 07/30 Read block 371 len 12288 08/30 Read block 248 len 4096 09/30 Flush 10/30 Write block 594 len 4096 data: 2 11/30 Write block 180 len 32768 data: 3 3 4 4 5 5 3 3 12/30 Write block 369 len 16384 data: 3 3 3 4 13/30 Read block 340 len 28672 14/30 Write block 249 len 16384 data: 2 2 2 2 15/30 Write block 324 len 8192 data: 3 3 16/30 Write block 305 len 40960 data: 2 2 2 2 2 2 2 2 2 2 17/30 Flush 18/30 Write block 58 len 28672 data: 5 6 5 5 4 5 4 19/30 Read block 259 len 4096 20/30 Read block 213 len 24576 21/30 Write block 396 len 36864 data: 4 4 5 5 4 4 3 3 2 22/30 Read block 423 len 8192 23/30 Flush 24/30 Write block 203 len 12288 data: 3 3 3 25/30 Write block 372 len 28672 data: 5 4 3 3 2 3 3 26/30 Read block 188 len 36864 27/30 Write block 479 len 12288 data: 2 2 2 28/30 Read block 107 len 4096 29/30 Write block 466 len 8192 data: 2 2 30/30 Write block 324 len 20480 data: 4 4 2 2 2 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 3 Done Done Done false 25 Acked 1024 Write 7 Done Done Done false 26 Acked 1025 Read 9 Done Done Done false 27 Acked 1026 Write 3 Done Done Done false 28 Acked 1027 Read 1 Done Done Done false 29 Acked 1028 Write 2 Sent Sent Done false 30 Acked 1029 Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 1 5 Done 5 5 6 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:28:34.350883044Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1238} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 19 19 19 F F F 1 020-039 14 14 14 24 24 24 F F F 2 040-059 14 14 14 26 26 26 F F F 3 060-079 14 14 14 26 26 26 F F F 4 080-099 10 10 10 18 18 18 F F F 5 100-119 9 9 9 16 16 16 F F F 6 120-139 14 14 14 24 24 24 F F F 7 140-159 10 10 10 17 17 17 F F F 8 160-179 14 14 14 23 23 23 F F F 9 180-199 14 14 14 25 25 25 F F F 10 200-219 12 12 12 21 21 21 T T T 11 220-239 6 6 6 12 12 12 F F F 12 240-259 14 14 14 25 25 25 F F F 13 260-279 12 12 12 21 21 21 F F F 14 280-299 5 5 5 11 11 11 F F F 15 300-319 14 14 14 25 25 25 F F F 16 320-339 14 14 14 25 25 25 T T T 17 340-359 10 10 10 19 19 19 F F F 18 360-379 14 14 14 25 25 25 T T T 19 380-399 14 14 14 26 26 26 F F F 20 400-419 14 14 14 26 26 26 F F F 21 420-439 12 12 12 21 21 21 F F F 22 440-459 12 12 12 21 21 21 F F F 23 460-479 1 1 1 1 1 1 T T T 24 480-499 8 8 8 15 15 15 T T T 25 500-519 14 14 14 22 22 22 F F F 26 520-539 12 12 12 21 21 21 F F F 27 540-559 11 11 11 20 20 20 F F F 28 560-579 12 12 12 21 21 21 F F F 29 580-599 14 14 14 25 25 25 F F F Max gen: 14, Max flush: 26 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 19 19 19 F F F 1 020-039 14 14 14 24 24 24 F F F 2 040-059 14 14 14 26 26 26 F F F 3 060-079 14 14 14 26 26 26 F F F 4 080-099 10 10 10 18 18 18 F F F 5 100-119 9 9 9 16 16 16 F F F 6 120-139 14 14 14 24 24 24 F F F 7 140-159 10 10 10 17 17 17 F F F 8 160-179 14 14 14 23 23 23 F F F 9 180-199 14 14 14 25 25 25 F F F 10 200-219 15 15 15 27 27 27 F F F 11 220-239 6 6 6 12 12 12 F F F 12 240-259 14 14 14 25 25 25 F F F 13 260-279 12 12 12 21 21 21 F F F 14 280-299 5 5 5 11 11 11 F F F 15 300-319 14 14 14 25 25 25 F F F 16 320-339 15 15 15 27 27 27 F F F 17 340-359 10 10 10 19 19 19 F F F 18 360-379 15 15 15 27 27 27 F F F 19 380-399 14 14 14 26 26 26 F F F 20 400-419 14 14 14 26 26 26 F F F 21 420-439 12 12 12 21 21 21 F F F 22 440-459 12 12 12 21 21 21 F F F 23 460-479 15 15 15 27 27 27 F F F 24 480-499 15 15 15 27 27 27 F F F 25 500-519 14 14 14 22 22 22 F F F 26 520-539 12 12 12 21 21 21 F F F 27 540-559 11 11 11 20 20 20 F F F 28 560-579 12 12 12 21 21 21 F F F 29 580-599 14 14 14 25 25 25 F F F Max gen: 15, Max flush: 27 Begin loop 8 Downstairs to restart: 2 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.714883111Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.715172005Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.715192283Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.715207513Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246} {"msg":"Crucible 4bee1e8c-920a-4876-a4c3-41e0175d2914 has session id: b66710f6-d079-4731-b399-e16f2a9ef538","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.715398972Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"Upstairs opts: Upstairs UUID: 4bee1e8c-920a-4876-a4c3-41e0175d2914, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.715451299Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"Crucible stats registered with UUID: 4bee1e8c-920a-4876-a4c3-41e0175d2914","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.715474958Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.746711558Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246} {"msg":"4bee1e8c-920a-4876-a4c3-41e0175d2914 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.746746907Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.746766895Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"client":"0","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.746811023Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"client":"0","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.746846821Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"client":"1","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.746881509Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"client":"1","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.746915557Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"client":"2","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.746945945Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"client":"2","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.746963644Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.746981303Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.747008332Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"ds_connection connected from Ok(127.0.0.1:55047)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.74704223Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"ds_connection connected from Ok(127.0.0.1:55503)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.747079708Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"ds_connection connected from Ok(127.0.0.1:44857)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.747116275Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.747875303Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"client":"0","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.747913461Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"client":"0","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.747941189Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"client":"2","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.747966857Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"client":"2","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.747992696Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"client":"1","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.748019215Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"client":"1","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.748529506Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"client":"0","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.748569503Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"client":"1","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.748601131Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"client":"2","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.749140021Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"client":"1","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.749168459Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"client":"1","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.749183878Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.749264374Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"client":"0","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.749298092Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"client":"0","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.749319311Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.74933635Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"client":"2","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.749350829Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"client":"2","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"[0]R flush_numbers[0..12]: [19, 24, 26, 26, 18, 16, 24, 17, 23, 25, 27, 12]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.749364688Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"":"downstairs","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"[0]R generation[0..12]: [10, 14, 14, 14, 10, 9, 14, 10, 14, 14, 15, 6]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.749378538Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"":"downstairs","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.749392507Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"":"downstairs","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"[1]R flush_numbers[0..12]: [19, 24, 26, 26, 18, 16, 24, 17, 23, 25, 27, 12]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.749406146Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"":"downstairs","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"[1]R generation[0..12]: [10, 14, 14, 14, 10, 9, 14, 10, 14, 14, 15, 6]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.749419645Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"":"downstairs","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.749433594Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"":"downstairs","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"[2]R flush_numbers[0..12]: [19, 24, 26, 26, 18, 16, 24, 17, 23, 25, 27, 12]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.749446983Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"":"downstairs","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"[2]R generation[0..12]: [10, 14, 14, 14, 10, 9, 14, 10, 14, 14, 15, 6]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.749460343Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"":"downstairs","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.749474092Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"":"downstairs","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-26T14:28:38.749486911Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"":"downstairs","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.74951142Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"":"downstairs","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"Next flush: 28","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.749535769Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"":"downstairs","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.749559917Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"":"downstairs","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.749585006Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.749607605Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"client":"0","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.749627733Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"client":"1","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.749646122Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"client":"2","":"downstairs","session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.749660002Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.749672401Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"4bee1e8c-920a-4876-a4c3-41e0175d2914 is now active with session: b66710f6-d079-4731-b399-e16f2a9ef538","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.74968531Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.749697329Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246,"session_id":"b66710f6-d079-4731-b399-e16f2a9ef538"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:38.749709589Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 140 len 32768 data: 2 2 2 2 2 2 2 2 03/30 Write block 507 len 24576 data: 2 2 3 3 3 3 04/30 Write block 595 len 20480 data: 2 2 2 2 2 05/30 Write block 331 len 16384 data: 2 2 3 3 06/30 Read block 92 len 20480 07/30 Write block 222 len 28672 data: 2 2 2 2 2 2 2 08/30 Write block 519 len 32768 data: 3 3 3 3 3 3 3 3 09/30 Write block 255 len 4096 data: 2 10/30 Write block 265 len 20480 data: 4 4 4 4 3 11/30 Write block 484 len 28672 data: 2 2 3 3 3 3 3 12/30 Flush 13/30 Flush 14/30 Write block 128 len 12288 data: 2 2 2 15/30 Write block 60 len 40960 data: 6 6 5 6 5 3 4 4 4 3 16/30 Flush 17/30 Write block 593 len 8192 data: 2 3 18/30 Read block 562 len 28672 19/30 Write block 338 len 8192 data: 2 2 20/30 Flush 21/30 Write block 338 len 12288 data: 3 3 2 22/30 Read block 73 len 36864 23/30 Read block 591 len 36864 24/30 Read block 542 len 28672 25/30 Read block 520 len 40960 26/30 Flush 27/30 Write block 346 len 40960 data: 2 2 2 2 2 3 3 3 3 3 28/30 Read block 279 len 32768 29/30 Write block 205 len 12288 data: 4 3 4 30/30 Read block 447 len 36864 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 10 Done Done Done false 28 Acked 1027 Read 8 Done Done Done false 29 Acked 1028 Write 3 Done Sent Sent false 30 Acked 1029 Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 4 2 2 8 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:28:38.772686518Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1246} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 19 19 19 F F F 1 020-039 14 14 14 24 24 24 F F F 2 040-059 14 14 14 26 26 26 F F F 3 060-079 16 16 16 31 31 31 F F F 4 080-099 10 10 10 18 18 18 F F F 5 100-119 9 9 9 16 16 16 F F F 6 120-139 16 16 16 31 31 31 F F F 7 140-159 16 16 16 29 29 29 F F F 8 160-179 14 14 14 23 23 23 F F F 9 180-199 14 14 14 25 25 25 F F F 10 200-219 15 15 15 27 27 27 T T T 11 220-239 16 16 16 29 29 29 F F F 12 240-259 16 16 16 29 29 29 F F F 13 260-279 16 16 16 29 29 29 F F F 14 280-299 5 5 5 11 11 11 F F F 15 300-319 14 14 14 25 25 25 F F F 16 320-339 16 16 16 33 33 33 F F F 17 340-359 16 16 16 33 33 33 T T T 18 360-379 15 15 15 27 27 27 F F F 19 380-399 14 14 14 26 26 26 F F F 20 400-419 14 14 14 26 26 26 F F F 21 420-439 12 12 12 21 21 21 F F F 22 440-459 12 12 12 21 21 21 F F F 23 460-479 15 15 15 27 27 27 F F F 24 480-499 16 16 16 29 29 29 F F F 25 500-519 16 16 16 29 29 29 F F F 26 520-539 16 16 16 29 29 29 F F F 27 540-559 11 11 11 20 20 20 F F F 28 560-579 12 12 12 21 21 21 F F F 29 580-599 16 16 16 32 32 32 F F F Max gen: 16, Max flush: 33 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 19 19 19 F F F 1 020-039 14 14 14 24 24 24 F F F 2 040-059 14 14 14 26 26 26 F F F 3 060-079 16 16 16 31 31 31 F F F 4 080-099 10 10 10 18 18 18 F F F 5 100-119 9 9 9 16 16 16 F F F 6 120-139 16 16 16 31 31 31 F F F 7 140-159 16 16 16 29 29 29 F F F 8 160-179 14 14 14 23 23 23 F F F 9 180-199 14 14 14 25 25 25 F F F 10 200-219 17 17 17 34 34 34 F F F 11 220-239 16 16 16 29 29 29 F F F 12 240-259 16 16 16 29 29 29 F F F 13 260-279 16 16 16 29 29 29 F F F 14 280-299 5 5 5 11 11 11 F F F 15 300-319 14 14 14 25 25 25 F F F 16 320-339 16 16 16 33 33 33 F F F 17 340-359 17 17 17 34 34 34 F F F 18 360-379 15 15 15 27 27 27 F F F 19 380-399 14 14 14 26 26 26 F F F 20 400-419 14 14 14 26 26 26 F F F 21 420-439 12 12 12 21 21 21 F F F 22 440-459 12 12 12 21 21 21 F F F 23 460-479 15 15 15 27 27 27 F F F 24 480-499 16 16 16 29 29 29 F F F 25 500-519 16 16 16 29 29 29 F F F 26 520-539 16 16 16 29 29 29 F F F 27 540-559 11 11 11 20 20 20 F F F 28 560-579 12 12 12 21 21 21 F F F 29 580-599 16 16 16 32 32 32 F F F Max gen: 17, Max flush: 34 Begin loop 9 Downstairs to restart: 2 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.076048679Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.076306245Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.076329743Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.076356332Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254} {"msg":"Crucible cc896acc-8458-4517-8f9e-51bf13b92a69 has session id: 17fde963-40c2-4b93-be4a-f745424e8466","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.076461306Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"Upstairs opts: Upstairs UUID: cc896acc-8458-4517-8f9e-51bf13b92a69, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.076509693Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"Crucible stats registered with UUID: cc896acc-8458-4517-8f9e-51bf13b92a69","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.076527452Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.107570234Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254} {"msg":"cc896acc-8458-4517-8f9e-51bf13b92a69 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.107600072Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.107619081Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"client":"0","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.107645619Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"client":"0","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.107662998Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"client":"1","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.107693147Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"client":"1","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.107724835Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"client":"2","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.107753053Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"client":"2","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.107775252Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.107798591Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.107829209Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"ds_connection connected from Ok(127.0.0.1:47231)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.107911455Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"ds_connection connected from Ok(127.0.0.1:59697)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.108104134Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"ds_connection connected from Ok(127.0.0.1:56572)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.108144321Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.108687351Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"client":"0","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.108713209Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"client":"0","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.108727778Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"client":"1","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.108740997Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"client":"1","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.108757307Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"client":"2","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.108778595Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"client":"2","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.109393081Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"client":"0","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.109420459Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"client":"2","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.109435328Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"client":"1","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.110056283Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"client":"1","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.110083752Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"client":"1","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.110099051Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.11011158Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"client":"2","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.110131379Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"client":"2","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.110144938Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.110157417Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"client":"0","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.110170867Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"client":"0","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"[0]R flush_numbers[0..12]: [19, 24, 26, 31, 18, 16, 31, 29, 23, 25, 34, 29]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.110184476Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"":"downstairs","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"[0]R generation[0..12]: [10, 14, 14, 16, 10, 9, 16, 16, 14, 14, 17, 16]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.110198415Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"":"downstairs","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.110212744Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"":"downstairs","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"[1]R flush_numbers[0..12]: [19, 24, 26, 31, 18, 16, 31, 29, 23, 25, 34, 29]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.110227393Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"":"downstairs","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"[1]R generation[0..12]: [10, 14, 14, 16, 10, 9, 16, 16, 14, 14, 17, 16]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.110256162Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"":"downstairs","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.11028595Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"":"downstairs","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"[2]R flush_numbers[0..12]: [19, 24, 26, 31, 18, 16, 31, 29, 23, 25, 34, 29]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.110303539Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"":"downstairs","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"[2]R generation[0..12]: [10, 14, 14, 16, 10, 9, 16, 16, 14, 14, 17, 16]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.110326568ZWait for a query_work_queue command to finish before sending IO ","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"":"downstairs","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.110355856Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"":"downstairs","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.110381045Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"":"downstairs","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.110406133Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"":"downstairs","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"Next flush: 35","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.110429622Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"":"downstairs","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.11045336Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"":"downstairs","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.110478949Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.110503098Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"client":"0","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.110519077Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"client":"1","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.110536466Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"client":"2","":"downstairs","session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-09-26T14:28:43.110559725Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.110581033Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"cc896acc-8458-4517-8f9e-51bf13b92a69 is now active with session: 17fde963-40c2-4b93-be4a-f745424e8466","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.110606462Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.110628041Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254,"session_id":"17fde963-40c2-4b93-be4a-f745424e8466"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:43.110649429Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 12 len 24576 data: 4 4 3 3 3 3 02/30 Write block 328 len 28672 data: 3 2 2 3 3 4 4 03/30 Read block 413 len 16384 04/30 Flush 05/30 Write block 355 len 20480 data: 4 2 2 2 2 06/30 Write block 204 len 4096 data: 4 07/30 Read block 459 len 12288 08/30 Read block 49 len 16384 09/30 Read block 276 len 28672 10/30 Write block 412 len 20480 data: 2 2 2 2 2 11/30 Write block 502 len 24576 data: 3 3 3 3 3 3 12/30 Read block 594 len 20480 13/30 Write block 191 len 8192 data: 4 4 14/30 Read block 380 len 36864 15/30 Read block 315 len 4096 16/30 Read block 380 len 8192 17/30 Read block 186 len 28672 18/30 Write block 23 len 8192 data: 4 5 19/30 Read block 487 len 16384 20/30 Write block 262 len 12288 data: 4 4 4 21/30 Write block 118 len 40960 data: 4 4 4 4 4 4 5 5 3 2 22/30 Read block 436 len 28672 23/30 Read block 144 len 20480 24/30 Write block 275 len 24576 data: 3 3 4 4 4 4 25/30 Read block 493 len 16384 26/30 Read block 579 len 12288 27/30 Write block 554 len 12288 data: 3 3 3 28/30 Read block 52 len 8192 29/30 Read block 592 len 32768 30/30 Read block 250 len 8192 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 5 Acked 1004 Write 5 Done Done Done false 6 Acked 1005 Write 1 Done Done Done false 7 Acked 1006 Read 3 Done Done Done false 8 Acked 1007 Read 4 Done Done Done false 9 Acked 1008 Read 7 Done Done Done false 10 Acked 1009 Write 5 Done Done Done false 11 Acked 1010 Write 6 Done Done Done false 12 Acked 1011 Read 5 Done Done Done false 13 Acked 1012 Write 2 Done Done Done false 14 Acked 1013 Read 9 Done Done Done false 15 Acked 1014 Read 1 Done Done Done false 16 Acked 1015 Read 2 Done Done Done false 17 Acked 1016 Read 7 Done Done Done false 18 Acked 1017 Write 2 Done Done Done false 19 Acked 1018 Read 4 Done Done Done false 20 Acked 1019 Write 3 Done Done Done false 21 Acked 1020 Write 10 Done Done Done false 22 Acked 1021 Read 7 Done Done Done false 23 Acked 1022 Read 5 Done Done Done false 24 Acked 1023 Write 6 Done Done Done false 25 Acked 1024 Read 4 Done Done Done false 26 Acked 1025 Read 3 Done Done Done false 27 Acked 1026 Write 3 Done Done Done false 28 Acked 1027 Read 2 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 26 26 26 78 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:28:43.131095942Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1254} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 35 35 35 F F F 1 020-039 14 14 14 24 24 24 T T T 2 040-059 14 14 14 26 26 26 F F F 3 060-079 16 16 16 31 31 31 F F F 4 080-099 10 10 10 18 18 18 F F F 5 100-119 9 9 9 16 16 16 T T T 6 120-139 16 16 16 31 31 31 T T T 7 140-159 16 16 16 29 29 29 F F F 8 160-179 14 14 14 23 23 23 F F F 9 180-199 14 14 14 25 25 25 T T T 10 200-219 17 17 17 34 34 34 T T T 11 220-239 16 16 16 29 29 29 F F F 12 240-259 16 16 16 29 29 29 F F F 13 260-279 16 16 16 29 29 29 T T T 14 280-299 5 5 5 11 11 11 T T T 15 300-319 14 14 14 25 25 25 F F F 16 320-339 18 18 18 35 35 35 F F F 17 340-359 17 17 17 34 34 34 T T T 18 360-379 15 15 15 27 27 27 F F F 19 380-399 14 14 14 26 26 26 F F F 20 400-419 14 14 14 26 26 26 T T T 21 420-439 12 12 12 21 21 21 F F F 22 440-459 12 12 12 21 21 21 F F F 23 460-479 15 15 15 27 27 27 F F F 24 480-499 16 16 16 29 29 29 F F F 25 500-519 16 16 16 29 29 29 T T T 26 520-539 16 16 16 29 29 29 F F F 27 540-559 11 11 11 20 20 20 T T T 28 560-579 12 12 12 21 21 21 F F F 29 580-599 16 16 16 32 32 32 F F F Max gen: 18, Max flush: 35 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 35 35 35 F F F 1 020-039 19 19 19 36 36 36 F F F 2 040-059 14 14 14 26 26 26 F F F 3 060-079 16 16 16 31 31 31 F F F 4 080-099 10 10 10 18 18 18 F F F 5 100-119 19 19 19 36 36 36 F F F 6 120-139 19 19 19 36 36 36 F F F 7 140-159 16 16 16 29 29 29 F F F 8 160-179 14 14 14 23 23 23 F F F 9 180-199 19 19 19 36 36 36 F F F 10 200-219 19 19 19 36 36 36 F F F 11 220-239 16 16 16 29 29 29 F F F 12 240-259 16 16 16 29 29 29 F F F 13 260-279 19 19 19 36 36 36 F F F 14 280-299 19 19 19 36 36 36 F F F 15 300-319 14 14 14 25 25 25 F F F 16 320-339 18 18 18 35 35 35 F F F 17 340-359 19 19 19 36 36 36 F F F 18 360-379 15 15 15 27 27 27 F F F 19 380-399 14 14 14 26 26 26 F F F 20 400-419 19 19 19 36 36 36 F F F 21 420-439 12 12 12 21 21 21 F F F 22 440-459 12 12 12 21 21 21 F F F 23 460-479 15 15 15 27 27 27 F F F 24 480-499 16 16 16 29 29 29 F F F 25 500-519 19 19 19 36 36 36 F F F 26 520-539 16 16 16 29 29 29 F F F 27 540-559 19 19 19 36 36 36 F F F 28 560-579 12 12 12 21 21 21 F F F 29 580-599 16 16 16 32 32 32 F F F Max gen: 19, Max flush: 36 Begin loop 10 Downstairs to restart: 2 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.52720315Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.52755596Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.527583149Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.527609347Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262} {"msg":"Crucible b2c4c6a7-649c-468e-a76f-2ece1803eb16 has session id: 1b90e8a4-ae26-4de8-b4f2-a7724999d081","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.527727621Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"Upstairs opts: Upstairs UUID: b2c4c6a7-649c-468e-a76f-2ece1803eb16, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.527772388Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"Crucible stats registered with UUID: b2c4c6a7-649c-468e-a76f-2ece1803eb16","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.527796937Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.55952839Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262} {"msg":"b2c4c6a7-649c-468e-a76f-2ece1803eb16 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.559564448Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.559582507Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"client":"0","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.559625014Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"client":"0","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.559660192Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"client":"1","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.559756487Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"client":"1","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.559796985Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"client":"2","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.559825163Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"client":"2","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.559850571Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.55987587Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.559901848Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"ds_connection connected from Ok(127.0.0.1:36626)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.560033311Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"ds_connection connected from Ok(127.0.0.1:40896)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.560069049Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"ds_connection connected from Ok(127.0.0.1:53230)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.560149374Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.560706533Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"client":"1","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.560732941Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"client":"1","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.560747551Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"client":"0","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.56076192Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"client":"0","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.560775619Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"client":"2","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.560789088Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"client":"2","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.561320498Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"client":"1","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.561359256Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"client":"0","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.561477929Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"client":"2","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.561910945Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"client":"0","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.561937583Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"client":"0","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.561953712Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.562034368Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"client":"1","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.562062556Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"client":"1","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.562077875Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.562090595Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"client":"2","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.562104444Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"client":"2","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"[0]R flush_numbers[0..12]: [35, 36, 26, 31, 18, 36, 36, 29, 23, 36, 36, 29]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.562118653Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"":"downstairs","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"[0]R generation[0..12]: [18, 19, 14, 16, 10, 19, 19, 16, 14, 19, 19, 16]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.562132512Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"":"downstairs","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.562146561Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"":"downstairs","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"[1]R flush_numbers[0..12]: [35, 36, 26, 31, 18, 36, 36, 29, 23, 36, 36, 29]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.562160321Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"":"downstairs","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"[1]R generation[0..12]: [18, 19, 14, 16, 10, 19, 19, 16, 14, 19, 19, 16]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.56217379Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"":"downstairs","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.562188019Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"":"downstairs","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"[2]R flush_numbers[0..12]: [35, 36, 26, 31, 18, 36, 36, 29, 23, 36, 36, 29]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.562205218Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"":"downstairs","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"[2]R generation[0..12]: [18, 19, 14, 16, 10, 19, 19, 16, 14, 19, 19, 16]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.562219397Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"":"downstairs","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.562234396Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"":"downstairs","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.562247266Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"":"downstairs","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.562260505Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"":"downstairs","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"Next flush: 37","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.562273594Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"":"downstairs","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.562286433Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"":"downstairs","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-26T14:28:47.562299403Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.562314982Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"client":"0","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.562328401Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"client":"1","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.56234147Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"client":"2","":"downstairs","session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.56235459Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.562366829Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"b2c4c6a7-649c-468e-a76f-2ece1803eb16 is now active with session: 1b90e8a4-ae26-4de8-b4f2-a7724999d081","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.562379758Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.562391738Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262,"session_id":"1b90e8a4-ae26-4de8-b4f2-a7724999d081"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:47.562406377Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 52 len 36864 data: 2 2 2 3 3 5 6 7 7 03/30 Write block 213 len 20480 data: 3 3 3 3 3 04/30 Write block 351 len 32768 data: 4 4 4 4 5 3 3 3 05/30 Read block 402 len 36864 06/30 Read block 503 len 40960 07/30 Write block 238 len 24576 data: 2 3 3 3 3 3 08/30 Read block 596 len 12288 09/30 Read block 304 len 24576 10/30 Read block 440 len 4096 11/30 Write block 224 len 20480 data: 3 3 3 3 3 12/30 Write block 500 len 12288 data: 3 3 4 13/30 Read block 365 len 24576 14/30 Read block 416 len 20480 15/30 Write block 564 len 12288 data: 2 2 2 16/30 Read block 234 len 12288 17/30 Read block 171 len 4096 18/30 Read block 63 len 32768 19/30 Write block 423 len 24576 data: 2 2 2 2 3 3 20/30 Write block 100 len 36864 data: 2 2 2 2 2 2 2 2 2 21/30 Read block 557 len 28672 22/30 Read block 366 len 4096 23/30 Write block 241 len 32768 data: 4 4 4 3 3 3 3 2 24/30 Write block 151 len 36864 data: 2 2 2 2 2 2 2 3 4 25/30 Read block 142 len 40960 26/30 Write block 443 len 4096 data: 2 27/30 Write block 346 len 12288 data: 3 3 3 28/30 Read block 143 len 8192 29/30 Write block 48 len 40960 data: 3 3 3 2 3 3 3 4 4 6 30/30 Read block 60 len 40960 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 2 Acked 1001 Write 9 Done Done Done false 3 Acked 1002 Write 5 Done Done Done false 4 Acked 1003 Write 8 Done Done Done false 5 Acked 1004 Read 9 Done Done Done false 6 Acked 1005 Read 10 Done Done Done false 7 Acked 1006 Write 6 Done Done Done false 8 Acked 1007 Read 3 Done Done Done false 9 Acked 1008 Read 6 Done Done Done false 10 Acked 1009 Read 1 Done Done Done false 11 Acked 1010 Write 5 Done Done Done false 12 Acked 1011 Write 3 Done Done Done false 13 Acked 1012 Read 6 Done Done Done false 14 Acked 1013 Read 5 Done Done Done false 15 Acked 1014 Write 3 Done Done Done false 16 Acked 1015 Read 3 Done Done Done false 17 Acked 1016 Read 1 Done Done Done false 18 Acked 1017 Read 8 Done Done Done false 19 Acked 1018 Write 6 Done Done Done false 20 Acked 1019 Write 9 Done Done Done false 21 Acked 1020 Read 7 Done Done Done false 22 Acked 1021 Read 1 Done Done Done false 23 Acked 1022 Write 8 Done Done Done false 24 Acked 1023 Write 9 Done Done Done false 25 Acked 1024 Read 10 Done Done Done false 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Write 3 Done Done Done false 28 Acked 1027 Read 2 Done Done Done false 29 Acked 1028 Write 10 Done Done Sent false 30 Acked 1029 Read 10 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 28 29 27 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1000 1000 1000 Downstairs last five completed: 1000 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:28:47.582327758Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1262} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 35 35 35 F F F 1 020-039 19 19 19 36 36 36 F F F 2 040-059 14 14 14 26 26 26 T T T 3 060-079 16 16 16 31 31 31 T T T 4 080-099 10 10 10 18 18 18 F F F 5 100-119 19 19 19 36 36 36 T T T 6 120-139 19 19 19 36 36 36 F F F 7 140-159 16 16 16 29 29 29 T T T 8 160-179 14 14 14 23 23 23 F F F 9 180-199 19 19 19 36 36 36 F F F 10 200-219 19 19 19 36 36 36 T T T 11 220-239 16 16 16 29 29 29 T T T 12 240-259 16 16 16 29 29 29 T T T 13 260-279 19 19 19 36 36 36 F F F 14 280-299 19 19 19 36 36 36 F F F 15 300-319 14 14 14 25 25 25 F F F 16 320-339 18 18 18 35 35 35 F F F 17 340-359 19 19 19 36 36 36 T T T 18 360-379 15 15 15 27 27 27 F F F 19 380-399 14 14 14 26 26 26 F F F 20 400-419 19 19 19 36 36 36 F F F 21 420-439 12 12 12 21 21 21 T T T 22 440-459 12 12 12 21 21 21 T T T 23 460-479 15 15 15 27 27 27 F F F 24 480-499 16 16 16 29 29 29 F F F 25 500-519 19 19 19 36 36 36 T T T 26 520-539 16 16 16 29 29 29 F F F 27 540-559 19 19 19 36 36 36 F F F 28 560-579 12 12 12 21 21 21 T T T 29 580-599 16 16 16 32 32 32 F F F Max gen: 19, Max flush: 36 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 35 35 35 F F F 1 020-039 19 19 19 36 36 36 F F F 2 040-059 20 20 20 37 37 37 F F F 3 060-079 20 20 20 37 37 37 F F F 4 080-099 10 10 10 18 18 18 F F F 5 100-119 20 20 20 37 37 37 F F F 6 120-139 19 19 19 36 36 36 F F F 7 140-159 20 20 20 37 37 37 F F F 8 160-179 14 14 14 23 23 23 F F F 9 180-199 19 19 19 36 36 36 F F F 10 200-219 20 20 20 37 37 37 F F F 11 220-239 20 20 20 37 37 37 F F F 12 240-259 20 20 20 37 37 37 F F F 13 260-279 19 19 19 36 36 36 F F F 14 280-299 19 19 19 36 36 36 F F F 15 300-319 14 14 14 25 25 25 F F F 16 320-339 18 18 18 35 35 35 F F F 17 340-359 20 20 20 37 37 37 F F F 18 360-379 15 15 15 27 27 27 F F F 19 380-399 14 14 14 26 26 26 F F F 20 400-419 19 19 19 36 36 36 F F F 21 420-439 20 20 20 37 37 37 F F F 22 440-459 20 20 20 37 37 37 F F F 23 460-479 15 15 15 27 27 27 F F F 24 480-499 16 16 16 29 29 29 F F F 25 500-519 20 20 20 37 37 37 F F F 26 520-539 16 16 16 29 29 29 F F F 27 540-559 19 19 19 36 36 36 F F F 28 560-579 20 20 20 37 37 37 F F F 29 580-599 16 16 16 32 32 32 F F F Max gen: 20, Max flush: 37 Begin loop 11 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.026754381Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.027104841Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.027133509Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.027159478Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270} {"msg":"Crucible 6ed5592b-d9ec-43d5-b5bc-d46a7939f4df has session id: 41a30e85-b060-4100-97d8-7e07d743f627","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.027180567Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"Upstairs opts: Upstairs UUID: 6ed5592b-d9ec-43d5-b5bc-d46a7939f4df, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.027219484Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"Crucible stats registered with UUID: 6ed5592b-d9ec-43d5-b5bc-d46a7939f4df","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.027242583Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.058658344Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270} {"msg":"6ed5592b-d9ec-43d5-b5bc-d46a7939f4df active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.058695142Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.058712551Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"client":"0","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.058740769Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"client":"0","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.058758488Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"client":"1","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.058775637Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"client":"1","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.058792026Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"client":"2","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.058813175Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"client":"2","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.058830274Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.058847673Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"":"io task","client":"0","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.058864562Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"ds_connection connected from Ok(127.0.0.1:53560)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.058961817Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"ds_connection connected from Ok(127.0.0.1:43178)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.059011124Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"":"io task","client":"0","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"ds_connection connected from Ok(127.0.0.1:45689)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.059046982Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.05978187Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"client":"2","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.059814189Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"client":"2","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.059829977Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"client":"0","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.059844037Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"client":"0","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.059857606Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"client":"1","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.059871865Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"client":"1","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.060416094Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"client":"2","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.06049571Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"client":"1","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.060533578Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"client":"0","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.060946604Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"client":"2","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.060982442Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"client":"2","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.061011311Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.061097716Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"client":"0","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.061116975Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"client":"0","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.061130734Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.061190411Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"client":"1","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.06120913Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"client":"1","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"[0]R flush_numbers[0..12]: [35, 36, 37, 37, 18, 37, 36, 37, 23, 36, 37, 37]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.061223539Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"":"downstairs","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"[0]R generation[0..12]: [18, 19, 20, 20, 10, 20, 19, 20, 14, 19, 20, 20]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.061237608Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"":"downstairs","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.061251867Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"":"downstairs","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"[1]R flush_numbers[0..12]: [35, 36, 37, 37, 18, 37, 36, 37, 23, 36, 37, 37]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.061265556Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"":"downstairs","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"[1]R generation[0..12]: [18, 19, 20, 20, 10, 20, 19, 20, 14, 19, 20, 20]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.061279066Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"":"downstairs","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.061293065Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"":"downstairs","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"[2]R flush_numbers[0..12]: [35, 36, 37, 37, 18, 37, 36, 37, 23, 36, 37, 37]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.061310194Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"":"downstairs","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"[2]R generation[0..12]: [18, 19, 20, 20, 10, 20, 19, 20, 14, 19, 20, 20]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.061324383Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"":"downstairs","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.061338532Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"":"downstairs","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"Max found gen is 21","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.061351571Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"":"downstairs","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"Generation requested: 22 >= found:21","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.061364531Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"":"downstairs","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"Next flush: 38","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.06137732Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"":"downstairs","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.061390229Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"":"downstairs","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.061404628Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.061420187Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"client":"0","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.061433687Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"client":"1","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.061446736Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"client":"2","":"downstairs","session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.061459985Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.061472244Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"6ed5592b-d9ec-43d5-b5bc-d46a7939f4df is now active with session: 41a30e85-b060-4100-97d8-7e07d743f627","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.061485224Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.061497163Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270,"session_id":"41a30e85-b060-4100-97d8-7e07d743f627"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:52.061511382Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 67 len 28672 data: 5 5 4 2 2 2 2 02/30 Write block 404 len 20480 data: 3 2 4 2 2 03/30 Read block 85 len 8192 04/30 Read block 136 len 24576 05/30 Write block 418 len 4096 data: 2 06/30 Read block 50 len 24576 07/30 Write block 259 len 24576 data: 3 3 4 5 5 5 08/30 Write block 178 len 4096 data: 3 09/30 Write block 26 len 8192 data: 4 4 10/30 Read block 581 len 20480 11/30 Write block 311 len 28672 data: 3 3 3 3 2 2 3 12/30 Read block 93 len 12288 13/30 Write block 422 len 40960 data: 2 3 3 3 3 4 4 3 3 3 14/30 Read block 193 len 28672 15/30 Read block 238 len 16384 16/30 Read block 129 len 40960 17/30 Read block 99 len 4096 18/30 Write block 63 len 36864 data: 7 6 4 5 6 6 5 3 3 19/30 Write block 396 len 28672 data: 5 5 6 6 5 5 4 20/30 Read block 527 len 36864 21/30 Read block 340 len 20480 22/30 Write block 121 len 8192 data: 5 5 23/30 Read block 236 len 24576 24/30 Read block 254 len 8192 25/30 Read block 279 len 28672 26/30 Flush 27/30 Write block 444 len 16384 data: 2 2 3 2 28/30 Write block 560 len 16384 data: 3 2 2 2 29/30 Read block 333 len 36864 30/30 Write block 449 len 36864 data: 3 3 3 3 3 3 4 4 4 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 4 Done Done Done false 28 Acked 1027 Write 4 Done Done Done false 29 Acked 1028 Read 9 Done Done Done false 30 Acked 1029 Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:28:52.083356455Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1270} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 35 35 35 F F F 1 020-039 22 22 22 38 38 38 F F F 2 040-059 20 20 20 37 37 37 F F F 3 060-079 22 22 22 38 38 38 F F F 4 080-099 10 10 10 18 18 18 F F F 5 100-119 20 20 20 37 37 37 F F F 6 120-139 22 22 22 38 38 38 F F F 7 140-159 20 20 20 37 37 37 F F F 8 160-179 22 22 22 38 38 38 F F F 9 180-199 19 19 19 36 36 36 F F F 10 200-219 20 20 20 37 37 37 F F F 11 220-239 20 20 20 37 37 37 F F F 12 240-259 22 22 22 38 38 38 F F F 13 260-279 22 22 22 38 38 38 F F F 14 280-299 19 19 19 36 36 36 F F F 15 300-319 22 22 22 38 38 38 F F F 16 320-339 18 18 18 35 35 35 F F F 17 340-359 20 20 20 37 37 37 F F F 18 360-379 15 15 15 27 27 27 F F F 19 380-399 22 22 22 38 38 38 F F F 20 400-419 22 22 22 38 38 38 F F F 21 420-439 22 22 22 38 38 38 F F F 22 440-459 20 20 20 37 37 37 T T T 23 460-479 15 15 15 27 27 27 F F F 24 480-499 16 16 16 29 29 29 F F F 25 500-519 20 20 20 37 37 37 F F F 26 520-539 16 16 16 29 29 29 F F F 27 540-559 19 19 19 36 36 36 F F F 28 560-579 20 20 20 37 37 37 T T T 29 580-599 16 16 16 32 32 32 F F F Max gen: 22, Max flush: 38 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 35 35 35 F F F 1 020-039 22 22 22 38 38 38 F F F 2 040-059 20 20 20 37 37 37 F F F 3 060-079 22 22 22 38 38 38 F F F 4 080-099 10 10 10 18 18 18 F F F 5 100-119 20 20 20 37 37 37 F F F 6 120-139 22 22 22 38 38 38 F F F 7 140-159 20 20 20 37 37 37 F F F 8 160-179 22 22 22 38 38 38 F F F 9 180-199 19 19 19 36 36 36 F F F 10 200-219 20 20 20 37 37 37 F F F 11 220-239 20 20 20 37 37 37 F F F 12 240-259 22 22 22 38 38 38 F F F 13 260-279 22 22 22 38 38 38 F F F 14 280-299 19 19 19 36 36 36 F F F 15 300-319 22 22 22 38 38 38 F F F 16 320-339 18 18 18 35 35 35 F F F 17 340-359 20 20 20 37 37 37 F F F 18 360-379 15 15 15 27 27 27 F F F 19 380-399 22 22 22 38 38 38 F F F 20 400-419 22 22 22 38 38 38 F F F 21 420-439 22 22 22 38 38 38 F F F 22 440-459 23 23 23 39 39 39 F F F 23 460-479 15 15 15 27 27 27 F F F 24 480-499 16 16 16 29 29 29 F F F 25 500-519 20 20 20 37 37 37 F F F 26 520-539 16 16 16 29 29 29 F F F 27 540-559 19 19 19 36 36 36 F F F 28 560-579 23 23 23 39 39 39 F F F 29 580-599 16 16 16 32 32 32 F F F Max gen: 23, Max flush: 39 Begin loop 12 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.392641462Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.392990483Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.393027961Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.393062568Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278} {"msg":"Crucible 42a4f616-b7d8-4c08-a7bd-1e1ca521b90a has session id: bd2082f1-275e-4572-b472-c49588d6206f","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.393134504Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"Upstairs opts: Upstairs UUID: 42a4f616-b7d8-4c08-a7bd-1e1ca521b90a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.393232089Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"Crucible stats registered with UUID: 42a4f616-b7d8-4c08-a7bd-1e1ca521b90a","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.393264507Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.424439771Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278} {"msg":"42a4f616-b7d8-4c08-a7bd-1e1ca521b90a active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.42447197Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.424495528Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"client":"0","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.424519217Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"client":"0","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.424536956Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"client":"1","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.424560465Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"client":"1","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.424631691Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"client":"2","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.424667478Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"client":"2","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.424688957Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.424713816Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"":"io task","client":"2","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.424739025Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"ds_connection connected from Ok(127.0.0.1:62349)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.424762353Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"ds_connection connected from Ok(127.0.0.1:58576)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.424787202Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"":"io task","client":"2","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"ds_connection connected from Ok(127.0.0.1:39606)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.424975001Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.425494052Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"client":"2","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.425579367Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"client":"2","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.425659882Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"client":"0","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.42569593Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"client":"0","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.425829423Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"client":"1","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.42587306Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"client":"1","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.426171463Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"client":"2","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.42641065Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"client":"0","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.42657605Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"client":"1","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.426777759Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"client":"2","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.426823707Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"client":"2","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.426857894Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.42692823Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"client":"0","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.426954589Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"client":"0","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.426976078Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.427092341Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"client":"1","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.42711153Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"client":"1","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"[0]R flush_numbers[0..12]: [35, 38, 37, 38, 18, 37, 38, 37, 38, 36, 37, 37]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.427133409Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"":"downstairs","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"[0]R generation[0..12]: [18, 22, 20, 22, 10, 20, 22, 20, 22, 19, 20, 20]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.427147628Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"":"downstairs","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.427161927Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"":"downstairs","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"[1]R flush_numbers[0..12]: [35, 38, 37, 38, 18, 37, 38, 37, 38, 36, 37, 37]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.427175647Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"":"downstairs","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"[1]R generation[0..12]: [18, 22, 20, 22, 10, 20, 22, 20, 22, 19, 20, 20]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.427189076Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"":"downstairs","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.427202865Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"":"downstairs","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"[2]R flush_numbers[0..12]: [35, 38, 37, 38, 18, 37, 38, 37, 38, 36, 37, 37]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.427216214Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"":"downstairs","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"[2]R generation[0..12]: [18, 22, 20, 22, 10, 20, 22, 20, 22, 19, 20, 20]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.427229684Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"":"downstairs","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.427243473Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"":"downstairs","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.427264222Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"":"downstairs","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.42728978Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"":"downstairs","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"Next flush: 40","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.427312939Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"":"downstairs","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":Wait for a query_work_queue command to finish before sending IO "2024-09-26T14:28:56.427338837Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"":"downstairs","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.427366916Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.427390194Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"client":"0","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.427414773Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"client":"1","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.427435252Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"client":"2","":"downstairs","session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.427449061Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.42747004Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"session_id":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"42a4f616-b7d8-4c08-a7bd-1e1ca521b90a is now active with session: bd2082f1-275e-4572-b472-c49588d6206f","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.427497578Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.427517927Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278,"session_id":"bd2082f1-275e-4572-b472-c49588d6206f"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:28:56.427538486Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 70 len 20480 data: 4 4 3 3 2 02/30 Read block 323 len 20480 03/30 Flush 04/30 Write block 498 len 24576 data: 3 3 4 4 5 4 05/30 Read block 555 len 24576 06/30 Read block 41 len 28672 07/30 Read block 310 len 20480 08/30 Read block 564 len 16384 09/30 Write block 475 len 36864 data: 2 2 2 2 3 3 3 2 2 10/30 Flush 11/30 Write block 68 len 4096 data: 7 12/30 Read block 96 len 28672 13/30 Read block 210 len 40960 14/30 Read block 486 len 40960 15/30 Write block 303 len 24576 data: 2 2 3 3 3 3 16/30 Write block 523 len 20480 data: 4 4 4 4 3 17/30 Write block 358 len 28672 data: 4 3 3 3 3 2 2 18/30 Read block 370 len 4096 19/30 Read block 0 len 8192 20/30 Flush 21/30 Write block 485 len 8192 data: 3 4 22/30 Write block 565 len 4096 data: 3 23/30 Read block 209 len 8192 24/30 Read block 471 len 32768 25/30 Read block 372 len 8192 26/30 Read block 491 len 16384 27/30 Read block 351 len 8192 28/30 Read block 392 len 4096 29/30 Read block 548 len 40960 30/30 Write block 298 len 12288 data: 3 3 3 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 2 Done Done Done false 22 Acked 1021 Write 1 Done Done Done false 23 Acked 1022 Read 2 Done Done Done false 24 Acked 1023 Read 8 Done Done Done false 25 Acked 1024 Read 2 Done Done Done false 26 Acked 1025 Read 4 Done Done Done false 27 Acked 1026 Read 2 Done Done Done false 28 Acked 1027 Read 1 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:28:56.451741045Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1278} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 35 35 35 F F F 1 020-039 22 22 22 38 38 38 F F F 2 040-059 20 20 20 37 37 37 F F F 3 060-079 24 24 24 42 42 42 F F F 4 080-099 10 10 10 18 18 18 F F F 5 100-119 20 20 20 37 37 37 F F F 6 120-139 22 22 22 38 38 38 F F F 7 140-159 20 20 20 37 37 37 F F F 8 160-179 22 22 22 38 38 38 F F F 9 180-199 19 19 19 36 36 36 F F F 10 200-219 20 20 20 37 37 37 F F F 11 220-239 20 20 20 37 37 37 F F F 12 240-259 22 22 22 38 38 38 F F F 13 260-279 22 22 22 38 38 38 F F F 14 280-299 19 19 19 36 36 36 T T T 15 300-319 24 24 24 42 42 42 T T T 16 320-339 18 18 18 35 35 35 F F F 17 340-359 24 24 24 42 42 42 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 22 22 22 38 38 38 F F F 20 400-419 22 22 22 38 38 38 F F F 21 420-439 22 22 22 38 38 38 F F F 22 440-459 23 23 23 39 39 39 F F F 23 460-479 24 24 24 41 41 41 F F F 24 480-499 24 24 24 41 41 41 T T T 25 500-519 24 24 24 41 41 41 F F F 26 520-539 24 24 24 42 42 42 F F F 27 540-559 19 19 19 36 36 36 F F F 28 560-579 23 23 23 39 39 39 T T T 29 580-599 16 16 16 32 32 32 F F F Max gen: 24, Max flush: 42 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 35 35 35 F F F 1 020-039 22 22 22 38 38 38 F F F 2 040-059 20 20 20 37 37 37 F F F 3 060-079 24 24 24 42 42 42 F F F 4 080-099 10 10 10 18 18 18 F F F 5 100-119 20 20 20 37 37 37 F F F 6 120-139 22 22 22 38 38 38 F F F 7 140-159 20 20 20 37 37 37 F F F 8 160-179 22 22 22 38 38 38 F F F 9 180-199 19 19 19 36 36 36 F F F 10 200-219 20 20 20 37 37 37 F F F 11 220-239 20 20 20 37 37 37 F F F 12 240-259 22 22 22 38 38 38 F F F 13 260-279 22 22 22 38 38 38 F F F 14 280-299 25 25 25 43 43 43 F F F 15 300-319 25 25 25 43 43 43 F F F 16 320-339 18 18 18 35 35 35 F F F 17 340-359 24 24 24 42 42 42 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 22 22 22 38 38 38 F F F 20 400-419 22 22 22 38 38 38 F F F 21 420-439 22 22 22 38 38 38 F F F 22 440-459 23 23 23 39 39 39 F F F 23 460-479 24 24 24 41 41 41 F F F 24 480-499 25 25 25 43 43 43 F F F 25 500-519 24 24 24 41 41 41 F F F 26 520-539 24 24 24 42 42 42 F F F 27 540-559 19 19 19 36 36 36 F F F 28 560-579 25 25 25 43 43 43 F F F 29 580-599 16 16 16 32 32 32 F F F Max gen: 25, Max flush: 43 Begin loop 13 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.819190579Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.819455204Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.819482502Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.819508561Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286} {"msg":"Crucible a5c77b1c-90f7-4757-aa78-57465641a950 has session id: 668a95e1-abec-405f-9cbd-cb643c6dd89d","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.819667642Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"Upstairs opts: Upstairs UUID: a5c77b1c-90f7-4757-aa78-57465641a950, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.819712629Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"Crucible stats registered with UUID: a5c77b1c-90f7-4757-aa78-57465641a950","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.819737248Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.851339028Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286} {"msg":"a5c77b1c-90f7-4757-aa78-57465641a950 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.851364567Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.851380806Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"client":"0","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.851405794Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"client":"0","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.851420573Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"client":"1","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.851433833Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"client":"1","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.851447042Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"client":"2","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.851460591Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"client":"2","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.85147426Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.851490679Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"":"io task","client":"0","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.851605263Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"":"io task","client":"1","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"ds_connection connected from Ok(127.0.0.1:40756)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.85165465Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"":"io task","client":"0","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"ds_connection connected from Ok(127.0.0.1:54912)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.851691658Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"ds_connection connected from Ok(127.0.0.1:58729)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.851952443Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"":"io task","client":"1","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.852343591Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"client":"2","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.852382989Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"client":"2","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.852445865Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"client":"1","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.852475693Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"client":"1","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.852695961Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"client":"0","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.852736719Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"client":"0","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.853013413Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"client":"2","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.853174154Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"client":"1","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.853511425Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"client":"0","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.853678976Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"client":"1","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.853706094Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"client":"1","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.853720773Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.853734052Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"client":"2","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.853747881Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"client":"2","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.853761191Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.853990978Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"client":"0","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.854014346Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"client":"0","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"[0]R flush_numbers[0..12]: [35, 38, 37, 42, 18, 37, 38, 37, 38, 36, 37, 37]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.854029565Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"":"downstairs","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"[0]R generation[0..12]: [18, 22, 20, 24, 10, 20, 22, 20, 22, 19, 20, 20]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.854043685Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"":"downstairs","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.854057884Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"":"downstairs","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"[1]R flush_numbers[0..12]: [35, 38, 37, 42, 18, 37, 38, 37, 38, 36, 37, 37]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.854075433Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"":"downstairs","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"[1]R generation[0..12]: [18, 22, 20, 24, 10, 20, 22, 20, 22, 19, 20, 20]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.854089692Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"":"downstairs","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.854103931Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"":"downstairs","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"[2]R flush_numbers[0..12]: [35, 38, 37, 42, 18, 37, 38, 37, 38, 36, 37, 37]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.85411788Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"":"downstairs","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"[2]R generation[0..12]: [18, 22, 20, 24, 10, 20, 22, 20, 22, 19, 20, 20]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.85413158Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"":"downstairs","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.854145769Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"":"downstairs","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.854159518Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"":"downstairs","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.854172617Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"":"downstairs","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"Next flush: 44","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.854185397Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"":"downstairs","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.854198316Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"":"downstairs","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.854215025Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.854226894Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"client":"0","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.854240234Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"client":"1","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.854253493Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"client":"2","":"downstairs","session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.854268852Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.854281411Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"a5c77b1c-90f7-4757-aa78-57465641a950 is now active with session: 668a95e1-abec-405f-9cbd-cb643c6dd89d","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.854294071Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.85430605Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286,"session_id":"668a95e1-abec-405f-9cbd-cb643c6dd89d"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:00.854318249Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 167 len 8192 data: 3 3 02/30 Read block 214 len 24576 03/30 Read block 264 len 20480 04/30 Write block 283 len 16384 data: 3 4 4 3 05/30 Read block 472 len 32768 06/30 Read block 250 len 28672 07/30 Write block 454 len 24576 data: 4 5 5 5 3 3 08/30 Read block 47 len 28672 09/30 Flush 10/30 Flush 11/30 Write block 295 len 16384 data: 3 3 3 4 12/30 Write block 545 len 28672 data: 3 3 2 2 2 2 3 13/30 Read block 500 len 40960 14/30 Flush 15/30 Write block 294 len 16384 data: 3 4 4 4 16/30 Write block 491 len 8192 data: 3 2 17/30 Write block 381 len 16384 data: 2 3 3 2 18/30 Write block 592 len 4096 data: 2 19/30 Write block 311 len 36864 data: 4 4 4 4 3 3 4 3 3 20/30 Read block 92 len 40960 21/30 Read block 467 len 4096 22/30 Write block 209 len 24576 data: 3 3 3 3 4 4 23/30 Write block 479 len 4096 data: 4 24/30 Read block 497 len 12288 25/30 Read block 279 len 32768 26/30 Write block 52 len 24576 data: 4 4 4 5 5 7 27/30 Read block 538 len 4096 28/30 Read block 321 len 40960 29/30 Read block 459 len 28672 30/30 Read block 140 len 12288 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 15 Acked 1014 Write 4 Done Done Done false 16 Acked 1015 Write 2 Done Done Done false 17 Acked 1016 Write 4 Done Done Done false 18 Acked 1017 Write 1 Done Done Done false 19 Acked 1018 Write 9 Done Done Done false 20 Acked 1019 Read 10 Done Done Done false 21 Acked 1020 Read 1 Done Done Done false 22 Acked 1021 Write 6 Done Done Done false 23 Acked 1022 Write 1 Done Done Done false 24 Acked 1023 Read 3 Done Done Done false 25 Acked 1024 Read 8 Done Done Done false 26 Acked 1025 Write 6 Done Done Done false 27 Acked 1026 Read 1 Done Done Done false 28 Acked 1027 Read 10 Done Done Done false 29 Acked 1028 Read 7 Done Done Done false 30 Acked 1029 Read 3 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 15 16 16 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:29:00.87830841Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1286} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 35 35 35 F F F 1 020-039 22 22 22 38 38 38 F F F 2 040-059 20 20 20 37 37 37 T T T 3 060-079 24 24 24 42 42 42 F F F 4 080-099 10 10 10 18 18 18 F F F 5 100-119 20 20 20 37 37 37 F F F 6 120-139 22 22 22 38 38 38 F F F 7 140-159 20 20 20 37 37 37 F F F 8 160-179 26 26 26 44 44 44 F F F 9 180-199 19 19 19 36 36 36 F F F 10 200-219 20 20 20 37 37 37 T T T 11 220-239 20 20 20 37 37 37 F F F 12 240-259 22 22 22 38 38 38 F F F 13 260-279 22 22 22 38 38 38 F F F 14 280-299 26 26 26 46 46 46 T T T 15 300-319 25 25 25 43 43 43 T T T 16 320-339 18 18 18 35 35 35 F F F 17 340-359 24 24 24 42 42 42 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 22 22 22 38 38 38 T T T 20 400-419 22 22 22 38 38 38 F F F 21 420-439 22 22 22 38 38 38 F F F 22 440-459 26 26 26 44 44 44 F F F 23 460-479 24 24 24 41 41 41 T T T 24 480-499 25 25 25 43 43 43 T T T 25 500-519 24 24 24 41 41 41 F F F 26 520-539 24 24 24 42 42 42 F F F 27 540-559 26 26 26 46 46 46 F F F 28 560-579 25 25 25 43 43 43 F F F 29 580-599 16 16 16 32 32 32 T T T Max gen: 26, Max flush: 46 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 35 35 35 F F F 1 020-039 22 22 22 38 38 38 F F F 2 040-059 27 27 27 47 47 47 F F F 3 060-079 24 24 24 42 42 42 F F F 4 080-099 10 10 10 18 18 18 F F F 5 100-119 20 20 20 37 37 37 F F F 6 120-139 22 22 22 38 38 38 F F F 7 140-159 20 20 20 37 37 37 F F F 8 160-179 26 26 26 44 44 44 F F F 9 180-199 19 19 19 36 36 36 F F F 10 200-219 27 27 27 47 47 47 F F F 11 220-239 20 20 20 37 37 37 F F F 12 240-259 22 22 22 38 38 38 F F F 13 260-279 22 22 22 38 38 38 F F F 14 280-299 27 27 27 47 47 47 F F F 15 300-319 27 27 27 47 47 47 F F F 16 320-339 18 18 18 35 35 35 F F F 17 340-359 24 24 24 42 42 42 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 27 27 27 47 47 47 F F F 20 400-419 22 22 22 38 38 38 F F F 21 420-439 22 22 22 38 38 38 F F F 22 440-459 26 26 26 44 44 44 F F F 23 460-479 27 27 27 47 47 47 F F F 24 480-499 27 27 27 47 47 47 F F F 25 500-519 24 24 24 41 41 41 F F F 26 520-539 24 24 24 42 42 42 F F F 27 540-559 26 26 26 46 46 46 F F F 28 560-579 25 25 25 43 43 43 F F F 29 580-599 27 27 27 47 47 47 F F F Max gen: 27, Max flush: 47 Begin loop 14 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.296715837Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.296983032Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.29700896Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.297040758Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294} {"msg":"Crucible b0316f1a-94ad-44c9-af2e-0b14cf2faf2f has session id: fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.29719664Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"Upstairs opts: Upstairs UUID: b0316f1a-94ad-44c9-af2e-0b14cf2faf2f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.297246707Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"Crucible stats registered with UUID: b0316f1a-94ad-44c9-af2e-0b14cf2faf2f","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.297271985Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.328487758Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294} {"msg":"b0316f1a-94ad-44c9-af2e-0b14cf2faf2f active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.328523975Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.328549854Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"client":"0","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.328595451Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"client":"0","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.3286313Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"client":"1","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.328720384Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"client":"1","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.32879408Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"client":"2","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.328836078Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"client":"2","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.328960301Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"":"io task","client":"2","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.328998519Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.329026707Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"":"io task","client":"1","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:48535)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.329053536Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"":"io task","client":"2","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:33268)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.329072575Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"":"io task","client":"1","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:43954)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.329086864Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.329440444Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"client":"2","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.329477161Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"client":"2","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.329581186Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"client":"0","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.329616644Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"client":"0","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.329745686Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"client":"1","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.329774925Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"client":"1","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.330099886Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"client":"0","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.330171812Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"client":"2","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.330554371Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"client":"1","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.33073406Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"client":"0","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.330754479Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"client":"0","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.330768799Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.330781508Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"client":"2","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.330795087Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"client":"2","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.330808236Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.33109035Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"client":"1","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.331108809Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"client":"1","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"[0]R flush_numbers[0..12]: [35, 38, 47, 42, 18, 37, 38, 37, 44, 36, 47, 37]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.331123128Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"":"downstairs","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"[0]R generation[0..12]: [18, 22, 27, 24, 10, 20, 22, 20, 26, 19, 27, 20]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.331137387Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"":"downstairs","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.331151397Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"":"downstairs","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"[1]R flush_numbers[0..12]: [35, 38, 47, 42, 18, 37, 38, 37, 44, 36, 47, 37]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.331164946Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"":"downstairs","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"[1]R generation[0..12]: [18, 22, 27, 24, 10, 20, 22, 20, 26, 19, 27, 20]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.331178535Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"":"downstairs","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.331192584Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"":"downstairs","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"[2]R flush_numbers[0..12]: [35, 38, 47, 42, 18, 37, 38, 37, 44, 36, 47, 37]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.331209654Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"":"downstairs","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"[2]R generation[0..12]: [18, 22, 27, 24, 10, 20, 22, 20, 26, 19, 27, 20]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.331223523Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"":"downstairs","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.331237542Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"":"downstairs","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.331250721Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"":"downstairs","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.33126389Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"":"downstairs","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"Next flush: 48","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.33127692Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"":"downstairs","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.331289869Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"":"downstairs","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-26T14:29:05.331303168Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.331318607Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"client":"0","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.331332167Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"client":"1","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.331345306Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"client":"2","":"downstairs","session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.331358635Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.331370934Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"b0316f1a-94ad-44c9-af2e-0b14cf2faf2f is now active with session: fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.331384113Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.331396273Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294,"session_id":"fcd7b0d5-d90f-4b55-8c04-1cfbee3f10b5"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:05.331408602Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 134 len 32768 data: 2 2 2 2 2 2 3 3 02/30 Read block 433 len 28672 03/30 Read block 560 len 36864 04/30 Write block 331 len 24576 data: 4 4 5 5 3 2 05/30 Write block 458 len 12288 data: 4 4 2 06/30 Read block 104 len 24576 07/30 Write block 198 len 4096 data: 3 08/30 Write block 170 len 4096 data: 3 09/30 Write block 480 len 16384 data: 4 4 3 3 10/30 Flush 11/30 Write block 550 len 4096 data: 3 12/30 Read block 371 len 24576 13/30 Read block 69 len 28672 14/30 Read block 136 len 8192 15/30 Read block 293 len 24576 16/30 Write block 582 len 16384 data: 2 3 2 2 17/30 Write block 313 len 32768 data: 5 5 4 4 5 4 4 3 18/30 Flush 19/30 Write block 291 len 20480 data: 3 3 3 4 5 20/30 Read block 101 len 8192 21/30 Read block 349 len 40960 22/30 Read block 443 len 24576 23/30 Write block 494 len 36864 data: 3 3 3 3 4 4 5 5 6 24/30 Flush 25/30 Write block 224 len 28672 data: 4 4 4 4 4 2 3 26/30 Read block 344 len 24576 27/30 Read block 593 len 12288 28/30 Read block 305 len 8192 29/30 Read block 528 len 20480 30/30 Write block 269 len 16384 data: 4 3 2 2 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 7 Done Done Done false 26 Acked 1025 Read 6 Done Done Done false 27 Acked 1026 Read 3 Done Done Done false 28 Acked 1027 Read 2 Done Done Done false 29 Acked 1028 Read 5 Done Done Done false 30 Acked 1029 Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:29:05.356407256Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1294} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 35 35 35 F F F 1 020-039 22 22 22 38 38 38 F F F 2 040-059 27 27 27 47 47 47 F F F 3 060-079 24 24 24 42 42 42 F F F 4 080-099 10 10 10 18 18 18 F F F 5 100-119 20 20 20 37 37 37 F F F 6 120-139 28 28 28 48 48 48 F F F 7 140-159 28 28 28 48 48 48 F F F 8 160-179 28 28 28 48 48 48 F F F 9 180-199 28 28 28 48 48 48 F F F 10 200-219 27 27 27 47 47 47 F F F 11 220-239 20 20 20 37 37 37 T T T 12 240-259 22 22 22 38 38 38 F F F 13 260-279 22 22 22 38 38 38 T T T 14 280-299 28 28 28 50 50 50 F F F 15 300-319 28 28 28 49 49 49 F F F 16 320-339 28 28 28 49 49 49 F F F 17 340-359 24 24 24 42 42 42 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 27 27 27 47 47 47 F F F 20 400-419 22 22 22 38 38 38 F F F 21 420-439 22 22 22 38 38 38 F F F 22 440-459 28 28 28 48 48 48 F F F 23 460-479 28 28 28 48 48 48 F F F 24 480-499 28 28 28 50 50 50 F F F 25 500-519 28 28 28 50 50 50 F F F 26 520-539 24 24 24 42 42 42 F F F 27 540-559 28 28 28 49 49 49 F F F 28 560-579 25 25 25 43 43 43 F F F 29 580-599 28 28 28 49 49 49 F F F Max gen: 28, Max flush: 50 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 35 35 35 F F F 1 020-039 22 22 22 38 38 38 F F F 2 040-059 27 27 27 47 47 47 F F F 3 060-079 24 24 24 42 42 42 F F F 4 080-099 10 10 10 18 18 18 F F F 5 100-119 20 20 20 37 37 37 F F F 6 120-139 28 28 28 48 48 48 F F F 7 140-159 28 28 28 48 48 48 F F F 8 160-179 28 28 28 48 48 48 F F F 9 180-199 28 28 28 48 48 48 F F F 10 200-219 27 27 27 47 47 47 F F F 11 220-239 29 29 29 51 51 51 F F F 12 240-259 22 22 22 38 38 38 F F F 13 260-279 29 29 29 51 51 51 F F F 14 280-299 28 28 28 50 50 50 F F F 15 300-319 28 28 28 49 49 49 F F F 16 320-339 28 28 28 49 49 49 F F F 17 340-359 24 24 24 42 42 42 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 27 27 27 47 47 47 F F F 20 400-419 22 22 22 38 38 38 F F F 21 420-439 22 22 22 38 38 38 F F F 22 440-459 28 28 28 48 48 48 F F F 23 460-479 28 28 28 48 48 48 F F F 24 480-499 28 28 28 50 50 50 F F F 25 500-519 28 28 28 50 50 50 F F F 26 520-539 24 24 24 42 42 42 F F F 27 540-559 28 28 28 49 49 49 F F F 28 560-579 25 25 25 43 43 43 F F F 29 580-599 28 28 28 49 49 49 F F F Max gen: 29, Max flush: 51 Begin loop 15 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.695082278Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.695430619Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.695466497Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.695498615Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302} {"msg":"Crucible fc474567-3b45-400b-845c-12acbc388811 has session id: e9d44370-547a-46db-af5a-84ca138d107e","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.695524124Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"Upstairs opts: Upstairs UUID: fc474567-3b45-400b-845c-12acbc388811, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.695575981Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"Crucible stats registered with UUID: fc474567-3b45-400b-845c-12acbc388811","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.695600729Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.726617832Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302} {"msg":"fc474567-3b45-400b-845c-12acbc388811 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.726649181Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.72666727Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"client":"0","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.726698048Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"client":"0","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.726716047Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"client":"1","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.726733166Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"client":"1","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.726749915Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"client":"2","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.726767724Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"client":"2","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.726785233Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.726816331Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.72684813Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"ds_connection connected from Ok(127.0.0.1:32837)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.726879618Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"ds_connection connected from Ok(127.0.0.1:39385)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.726912676Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"ds_connection connected from Ok(127.0.0.1:40368)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.726952174Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.727806905Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"client":"0","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.727835074Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"client":"0","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.727850333Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"client":"1","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.727864372Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"client":"1","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.727877821Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"client":"2","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.72789122Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"client":"2","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.728544523Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"client":"1","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.728572852Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"client":"2","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.72860362Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"client":"0","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.72913787Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"client":"1","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.729165508Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"client":"1","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.729180158Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.729242634Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"client":"0","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.729263243Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"client":"0","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.729280522Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.729293321Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"client":"2","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.72930693Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"client":"2","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"[0]R flush_numbers[0..12]: [35, 38, 47, 42, 18, 37, 48, 48, 48, 48, 47, 51]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.72932081Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"":"downstairs","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"[0]R generation[0..12]: [18, 22, 27, 24, 10, 20, 28, 28, 28, 28, 27, 29]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.729334659Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"":"downstairs","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.729348918Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"":"downstairs","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"[1]R flush_numbers[0..12]: [35, 38, 47, 42, 18, 37, 48, 48, 48, 48, 47, 51]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.729362677Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"":"downstairs","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"[1]R generation[0..12]: [18, 22, 27, 24, 10, 20, 28, 28, 28, 28, 27, 29]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.729376606Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"":"downstairs","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.729390706Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"":"downstairs","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"[2]R flush_numbers[0..12]: [35, 38, 47, 42, 18, 37, 48, 48, 48, 48, 47, 51]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.729404165Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"":"downstairs","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"[2]R generation[0..12]: [18, 22, 27, 24, 10, 20, 28, 28, 28, 28, 27, 29]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.729417704Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"":"downstairs","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.729431633Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"":"downstairs","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.729444492Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"":"downstairs","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.729457742Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"":"downstairs","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"Next flush: 52","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.729472431Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"":"downstairs","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.72948568Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","Wait for a query_work_queue command to finish before sending IO pid":1302,"":"downstairs","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.729502899Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.729514979Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"client":"0","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.729528198Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"client":"1","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.729541287Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"client":"2","":"downstairs","session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.729554316Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.729566616Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"fc474567-3b45-400b-845c-12acbc388811 is now active with session: e9d44370-547a-46db-af5a-84ca138d107e","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.729579435Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.729591314Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302,"session_id":"e9d44370-547a-46db-af5a-84ca138d107e"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:09.729603594Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 50 len 16384 data: 4 3 5 5 02/30 Read block 358 len 32768 03/30 Write block 338 len 36864 data: 4 4 3 2 3 3 3 3 4 04/30 Read block 443 len 8192 05/30 Write block 302 len 36864 data: 2 3 3 4 4 4 4 3 3 06/30 Read block 581 len 28672 07/30 Read block 452 len 28672 08/30 Read block 406 len 24576 09/30 Write block 530 len 40960 data: 3 3 3 4 4 3 4 3 3 3 10/30 Write block 570 len 36864 data: 3 3 3 3 3 2 2 2 2 11/30 Write block 444 len 20480 data: 3 3 4 3 2 12/30 Write block 499 len 40960 data: 5 6 6 7 5 4 4 4 4 3 13/30 Write block 49 len 4096 data: 4 14/30 Read block 531 len 20480 15/30 Read block 560 len 20480 16/30 Read block 295 len 36864 17/30 Read block 133 len 20480 18/30 Read block 215 len 12288 19/30 Flush 20/30 Write block 112 len 16384 data: 3 3 3 2 21/30 Write block 209 len 12288 data: 4 4 4 22/30 Read block 551 len 28672 23/30 Write block 92 len 8192 data: 2 2 24/30 Read block 412 len 16384 25/30 Read block 139 len 4096 26/30 Write block 98 len 4096 data: 2 27/30 Write block 22 len 28672 data: 4 5 6 4 5 5 4 28/30 Read block 34 len 40960 29/30 Write block 0 len 32768 data: 3 3 2 2 2 2 2 2 30/30 Write block 64 len 40960 data: 7 5 6 7 8 6 5 5 4 4 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 20 Acked 1019 Write 4 Done Done Done false 21 Acked 1020 Write 3 Done Done Done false 22 Acked 1021 Read 7 Done Done Done false 23 Acked 1022 Write 2 Done Done Done false 24 Acked 1023 Read 4 Done Done Done false 25 Acked 1024 Read 1 Done Done Done false 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Write 7 Done Done Done false 28 Acked 1027 Read 10 Done Done Done false 29 Acked 1028 Write 8 Sent Sent Sent false 30 Acked 1029 Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:29:09.752077531Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1302} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 35 35 35 T T T 1 020-039 22 22 22 38 38 38 T T T 2 040-059 30 30 30 52 52 52 F F F 3 060-079 24 24 24 42 42 42 T T T 4 080-099 10 10 10 18 18 18 T T T 5 100-119 20 20 20 37 37 37 T T T 6 120-139 28 28 28 48 48 48 F F F 7 140-159 28 28 28 48 48 48 F F F 8 160-179 28 28 28 48 48 48 F F F 9 180-199 28 28 28 48 48 48 F F F 10 200-219 27 27 27 47 47 47 T T T 11 220-239 29 29 29 51 51 51 F F F 12 240-259 22 22 22 38 38 38 F F F 13 260-279 29 29 29 51 51 51 F F F 14 280-299 28 28 28 50 50 50 F F F 15 300-319 30 30 30 52 52 52 F F F 16 320-339 30 30 30 52 52 52 F F F 17 340-359 30 30 30 52 52 52 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 27 27 27 47 47 47 F F F 20 400-419 22 22 22 38 38 38 F F F 21 420-439 22 22 22 38 38 38 F F F 22 440-459 30 30 30 52 52 52 F F F 23 460-479 28 28 28 48 48 48 F F F 24 480-499 30 30 30 52 52 52 F F F 25 500-519 30 30 30 52 52 52 F F F 26 520-539 30 30 30 52 52 52 F F F 27 540-559 28 28 28 49 49 49 F F F 28 560-579 30 30 30 52 52 52 F F F 29 580-599 28 28 28 49 49 49 F F F Max gen: 30, Max flush: 52 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 53 53 53 F F F 1 020-039 31 31 31 53 53 53 F F F 2 040-059 30 30 30 52 52 52 F F F 3 060-079 31 31 31 53 53 53 F F F 4 080-099 31 31 31 53 53 53 F F F 5 100-119 31 31 31 53 53 53 F F F 6 120-139 28 28 28 48 48 48 F F F 7 140-159 28 28 28 48 48 48 F F F 8 160-179 28 28 28 48 48 48 F F F 9 180-199 28 28 28 48 48 48 F F F 10 200-219 31 31 31 53 53 53 F F F 11 220-239 29 29 29 51 51 51 F F F 12 240-259 22 22 22 38 38 38 F F F 13 260-279 29 29 29 51 51 51 F F F 14 280-299 28 28 28 50 50 50 F F F 15 300-319 30 30 30 52 52 52 F F F 16 320-339 30 30 30 52 52 52 F F F 17 340-359 30 30 30 52 52 52 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 27 27 27 47 47 47 F F F 20 400-419 22 22 22 38 38 38 F F F 21 420-439 22 22 22 38 38 38 F F F 22 440-459 30 30 30 52 52 52 F F F 23 460-479 28 28 28 48 48 48 F F F 24 480-499 30 30 30 52 52 52 F F F 25 500-519 30 30 30 52 52 52 F F F 26 520-539 30 30 30 52 52 52 F F F 27 540-559 28 28 28 49 49 49 F F F 28 560-579 30 30 30 52 52 52 F F F 29 580-599 28 28 28 49 49 49 F F F Max gen: 31, Max flush: 53 Begin loop 16 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.177255634Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.17750785Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.177532428Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.177551117Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310} {"msg":"Crucible 95dc9410-2518-4018-864e-4cfb4ffe767d has session id: 08ddd434-4f62-461b-8b3d-f39af24fd6f6","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.177745836Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"Upstairs opts: Upstairs UUID: 95dc9410-2518-4018-864e-4cfb4ffe767d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.177808733Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"Crucible stats registered with UUID: 95dc9410-2518-4018-864e-4cfb4ffe767d","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.177836251Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.209122959Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310} {"msg":"95dc9410-2518-4018-864e-4cfb4ffe767d active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.209155117Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.209172896Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"client":"0","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.209200025Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"client":"0","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.209217634Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"client":"1","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.209234803Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"client":"1","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.209251662Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"client":"2","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.209268501Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"client":"2","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.20928488Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.209315608Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"":"io task","client":"0","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.209348496Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:56277)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.209379435Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:65245)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.209412523Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"":"io task","client":"0","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:42262)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.209625081Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.210213967Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"client":"0","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.210241116Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"client":"0","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.210258955Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"client":"1","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.210272874Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"client":"1","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.210286093Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"client":"2","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.210299273Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"client":"2","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.21087788Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"client":"1","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.211012962Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"client":"0","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.2110421Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"client":"2","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.211478466Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"client":"1","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.211507224Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"client":"1","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.211522803Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.211598419Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"client":"2","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.211618588Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"client":"2","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.211632487Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.211695394Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"client":"0","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.211714752Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"client":"0","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"[0]R flush_numbers[0..12]: [53, 53, 52, 53, 53, 53, 48, 48, 48, 48, 53, 51]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.211729832Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"":"downstairs","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"[0]R generation[0..12]: [31, 31, 30, 31, 31, 31, 28, 28, 28, 28, 31, 29]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.2117474Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"":"downstairs","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.21176238Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"":"downstairs","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"[1]R flush_numbers[0..12]: [53, 53, 52, 53, 53, 53, 48, 48, 48, 48, 53, 51]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.211776219Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"":"downstairs","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"[1]R generation[0..12]: [31, 31, 30, 31, 31, 31, 28, 28, 28, 28, 31, 29]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.211790928Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"":"downstairs","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.211805037Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"":"downstairs","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"[2]R flush_numbers[0..12]: [53, 53, 52, 53, 53, 53, 48, 48, 48, 48, 53, 51]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.211818547Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"":"downstairs","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"[2]R generation[0..12]: [31, 31, 30, 31, 31, 31, 28, 28, 28, 28, 31, 29]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.211832286Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"":"downstairs","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.211846155Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"":"downstairs","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.211858994Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"":"downstairs","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.211872213Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"":"downstairs","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"Next flush: 54","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.211885253Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"":"downstairs","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.211898012Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"":"downstairs","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"No downstairs reconciliation required","vWait for a query_work_queue command to finish before sending IO ":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.211914211Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.21192628Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"client":"0","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.21194176Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"client":"1","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.211954999Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"client":"2","":"downstairs","session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.211968598Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.211980857Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"95dc9410-2518-4018-864e-4cfb4ffe767d is now active with session: 08ddd434-4f62-461b-8b3d-f39af24fd6f6","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.211993566Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.212023055Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310,"session_id":"08ddd434-4f62-461b-8b3d-f39af24fd6f6"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:14.212035524Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 233 len 40960 data: 2 2 2 2 2 3 4 4 5 5 02/30 Write block 290 len 28672 data: 4 4 4 4 5 6 5 03/30 Write block 281 len 28672 data: 3 3 4 5 5 4 3 04/30 Write block 382 len 16384 data: 4 4 3 3 05/30 Write block 575 len 36864 data: 3 3 3 3 2 2 2 3 4 06/30 Flush 07/30 Write block 55 len 12288 data: 6 6 8 08/30 Write block 465 len 8192 data: 2 3 09/30 Flush 10/30 Write block 454 len 32768 data: 5 6 6 6 5 5 3 2 11/30 Flush 12/30 Write block 80 len 32768 data: 2 2 2 2 2 2 3 3 13/30 Write block 217 len 32768 data: 4 3 3 2 2 3 3 5 14/30 Read block 226 len 40960 15/30 Read block 196 len 20480 16/30 Read block 246 len 16384 17/30 Write block 261 len 32768 data: 5 6 6 6 5 5 5 5 18/30 Read block 29 len 16384 19/30 Read block 535 len 20480 20/30 Read block 297 len 20480 21/30 Flush 22/30 Write block 309 len 8192 data: 4 4 23/30 Read block 76 len 24576 24/30 Read block 111 len 4096 25/30 Flush 26/30 Write block 327 len 20480 data: 3 4 3 3 5 27/30 Read block 515 len 4096 28/30 Read block 119 len 8192 29/30 Write block 428 len 32768 data: 5 4 4 4 4 4 3 3 30/30 Read block 34 len 4096 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 5 Done Done Done false 27 Acked 1026 Read 1 Done Done Done false 28 Acked 1027 Read 2 Done Done Done false 29 Acked 1028 Write 8 Sent Done Sent false 30 Acked 1029 Read 1 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 2 4 Done 3 5 3 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:29:14.239399265Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1310} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 53 53 53 F F F 1 020-039 31 31 31 53 53 53 F F F 2 040-059 32 32 32 55 55 55 F F F 3 060-079 31 31 31 53 53 53 F F F 4 080-099 32 32 32 57 57 57 F F F 5 100-119 31 31 31 53 53 53 F F F 6 120-139 28 28 28 48 48 48 F F F 7 140-159 28 28 28 48 48 48 F F F 8 160-179 28 28 28 48 48 48 F F F 9 180-199 28 28 28 48 48 48 F F F 10 200-219 32 32 32 57 57 57 F F F 11 220-239 32 32 32 57 57 57 F F F 12 240-259 32 32 32 54 54 54 F F F 13 260-279 32 32 32 57 57 57 F F F 14 280-299 32 32 32 54 54 54 F F F 15 300-319 32 32 32 58 58 58 F F F 16 320-339 30 30 30 52 52 52 T T T 17 340-359 30 30 30 52 52 52 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 32 32 32 54 54 54 F F F 20 400-419 22 22 22 38 38 38 F F F 21 420-439 22 22 22 38 38 38 T T T 22 440-459 32 32 32 56 56 56 F F F 23 460-479 32 32 32 56 56 56 F F F 24 480-499 30 30 30 52 52 52 F F F 25 500-519 30 30 30 52 52 52 F F F 26 520-539 30 30 30 52 52 52 F F F 27 540-559 28 28 28 49 49 49 F F F 28 560-579 32 32 32 54 54 54 F F F 29 580-599 32 32 32 54 54 54 F F F Max gen: 32, Max flush: 58 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 53 53 53 F F F 1 020-039 31 31 31 53 53 53 F F F 2 040-059 32 32 32 55 55 55 F F F 3 060-079 31 31 31 53 53 53 F F F 4 080-099 32 32 32 57 57 57 F F F 5 100-119 31 31 31 53 53 53 F F F 6 120-139 28 28 28 48 48 48 F F F 7 140-159 28 28 28 48 48 48 F F F 8 160-179 28 28 28 48 48 48 F F F 9 180-199 28 28 28 48 48 48 F F F 10 200-219 32 32 32 57 57 57 F F F 11 220-239 32 32 32 57 57 57 F F F 12 240-259 32 32 32 54 54 54 F F F 13 260-279 32 32 32 57 57 57 F F F 14 280-299 32 32 32 54 54 54 F F F 15 300-319 32 32 32 58 58 58 F F F 16 320-339 33 33 33 59 59 59 F F F 17 340-359 30 30 30 52 52 52 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 32 32 32 54 54 54 F F F 20 400-419 22 22 22 38 38 38 F F F 21 420-439 33 33 33 59 59 59 F F F 22 440-459 32 32 32 56 56 56 F F F 23 460-479 32 32 32 56 56 56 F F F 24 480-499 30 30 30 52 52 52 F F F 25 500-519 30 30 30 52 52 52 F F F 26 520-539 30 30 30 52 52 52 F F F 27 540-559 28 28 28 49 49 49 F F F 28 560-579 32 32 32 54 54 54 F F F 29 580-599 32 32 32 54 54 54 F F F Max gen: 33, Max flush: 59 Begin loop 17 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.581439256Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.581656284Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.581674133Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.581689662Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318} {"msg":"Crucible 2664af21-2a06-476f-9d45-c66599e3f55d has session id: 94997209-10d3-4d3d-bb90-f4001658dcff","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.581863112Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"Upstairs opts: Upstairs UUID: 2664af21-2a06-476f-9d45-c66599e3f55d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.58191175Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"Crucible stats registered with UUID: 2664af21-2a06-476f-9d45-c66599e3f55d","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.581937828Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.613572237Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318} {"msg":"2664af21-2a06-476f-9d45-c66599e3f55d active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.613608684Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.613635643Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"client":"0","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.6136806Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"client":"0","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.613714878Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"client":"1","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.613748077Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"client":"1","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.613778885Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"client":"2","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.613813913Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"client":"2","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.613847041Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.613878919Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"":"io task","client":"0","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.613907987Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"":"io task","client":"1","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"ds_connection connected from Ok(127.0.0.1:50947)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.613937586Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"ds_connection connected from Ok(127.0.0.1:50297)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.614050599Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"":"io task","client":"1","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"ds_connection connected from Ok(127.0.0.1:59144)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.614093167Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"":"io task","client":"0","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.614637926Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"client":"2","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.614664505Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"client":"2","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.614775398Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"client":"1","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.614851344Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"client":"1","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.614878273Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"client":"0","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.614893392Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"client":"0","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.615342196Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"client":"2","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.615380824Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"client":"0","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.615492418Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"client":"1","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.615894715Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"client":"0","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.615932963Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"client":"0","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.615958131Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.61598174Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"client":"2","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.616015458Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"client":"2","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.616031027Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.616043707Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"client":"1","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.616057656Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"client":"1","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"[0]R flush_numbers[0..12]: [53, 53, 55, 53, 57, 53, 48, 48, 48, 48, 57, 57]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.616071995Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"":"downstairs","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"[0]R generation[0..12]: [31, 31, 32, 31, 32, 31, 28, 28, 28, 28, 32, 32]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.616085934Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"":"downstairs","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.616100103Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"":"downstairs","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"[1]R flush_numbers[0..12]: [53, 53, 55, 53, 57, 53, 48, 48, 48, 48, 57, 57]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.616113743Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"":"downstairs","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"[1]R generation[0..12]: [31, 31, 32, 31, 32, 31, 28, 28, 28, 28, 32, 32]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.616127572Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"":"downstairs","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.616141721Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"":"downstairs","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"[2]R flush_numbers[0..12]: [53, 53, 55, 53, 57, 53, 48, 48, 48, 48, 57, 57]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.616159Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"":"downstairs","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"[2]R generation[0..12]: [31, 31, 32, 31, 32, 31, 28, 28, 28, 28, 32, 32]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.616172959Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"":"downstairs","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.616186989Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"":"downstairs","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.616200068Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"":"downstairs","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.616213237Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"":"downstairs","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"Next flush: 60","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.616226156Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"":"downstairs","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.616239055Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"":"downstairs","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.616252385Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.616267884Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"client":"0","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.616281193Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"client":"1","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.616294263Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"client":"2","":"downstairs","session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.616307362Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.616319581Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"2664af21-2a06-476f-9d45-c66599e3f55d is now active with session: 94997209-10d3-4d3d-bb90-f4001658dcff","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.61633324Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"session_id":"94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.616345089Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318,"session_id":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 94997209-10d3-4d3d-bb90-f4001658dcff"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:18.616363699Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 183 len 24576 data: 5 6 6 4 4 4 02/30 Read block 280 len 8192 03/30 Write block 113 len 20480 data: 4 4 3 2 2 04/30 Write block 23 len 24576 data: 6 7 5 6 6 5 05/30 Read block 14 len 32768 06/30 Read block 13 len 32768 07/30 Flush 08/30 Write block 108 len 32768 data: 3 2 2 3 4 5 5 4 09/30 Read block 520 len 28672 10/30 Write block 395 len 16384 data: 4 6 6 7 11/30 Read block 256 len 36864 12/30 Write block 151 len 8192 data: 3 3 13/30 Write block 538 len 4096 data: 4 14/30 Write block 471 len 12288 data: 2 2 2 15/30 Write block 282 len 36864 data: 4 5 6 6 5 4 4 5 5 16/30 Write block 127 len 32768 data: 3 3 3 3 2 2 2 3 17/30 Read block 144 len 12288 18/30 Read block 253 len 36864 19/30 Read block 241 len 28672 20/30 Write block 202 len 20480 data: 4 4 5 5 4 21/30 Read block 149 len 36864 22/30 Read block 556 len 12288 23/30 Read block 321 len 12288 24/30 Read block 329 len 36864 25/30 Read block 290 len 24576 26/30 Flush 27/30 Write block 192 len 36864 data: 5 4 4 3 3 3 4 3 4 28/30 Write block 548 len 4096 data: 3 29/30 Read block 588 len 8192 30/30 Read block 174 len 12288 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 9 Done Done Sent false 28 Acked 1027 Write 1 Done Done Sent false 29 Acked 1028 Read 2 Done Done Sent false 30 Acked 1029 Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 4 5 Done 3 4 0 7 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:29:18.640216948Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1318} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 53 53 53 F F F 1 020-039 34 34 34 60 60 60 F F F 2 040-059 32 32 32 55 55 55 F F F 3 060-079 31 31 31 53 53 53 F F F 4 080-099 32 32 32 57 57 57 F F F 5 100-119 34 34 34 61 61 61 F F F 6 120-139 34 34 34 61 61 61 F F F 7 140-159 34 34 34 61 61 61 F F F 8 160-179 28 28 28 48 48 48 F F F 9 180-199 34 34 34 60 60 60 T T T 10 200-219 34 34 34 61 61 61 T T T 11 220-239 32 32 32 57 57 57 F F F 12 240-259 32 32 32 54 54 54 F F F 13 260-279 32 32 32 57 57 57 F F F 14 280-299 34 34 34 61 61 61 F F F 15 300-319 32 32 32 58 58 58 F F F 16 320-339 33 33 33 59 59 59 F F F 17 340-359 30 30 30 52 52 52 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 34 34 34 61 61 61 F F F 20 400-419 22 22 22 38 38 38 F F F 21 420-439 33 33 33 59 59 59 F F F 22 440-459 32 32 32 56 56 56 F F F 23 460-479 34 34 34 61 61 61 F F F 24 480-499 30 30 30 52 52 52 F F F 25 500-519 30 30 30 52 52 52 F F F 26 520-539 34 34 34 61 61 61 F F F 27 540-559 28 28 28 49 49 49 T T T 28 560-579 32 32 32 54 54 54 F F F 29 580-599 32 32 32 54 54 54 F F F Max gen: 34, Max flush: 61 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 53 53 53 F F F 1 020-039 34 34 34 60 60 60 F F F 2 040-059 32 32 32 55 55 55 F F F 3 060-079 31 31 31 53 53 53 F F F 4 080-099 32 32 32 57 57 57 F F F 5 100-119 34 34 34 61 61 61 F F F 6 120-139 34 34 34 61 61 61 F F F 7 140-159 34 34 34 61 61 61 F F F 8 160-179 28 28 28 48 48 48 F F F 9 180-199 35 35 35 62 62 62 F F F 10 200-219 35 35 35 62 62 62 F F F 11 220-239 32 32 32 57 57 57 F F F 12 240-259 32 32 32 54 54 54 F F F 13 260-279 32 32 32 57 57 57 F F F 14 280-299 34 34 34 61 61 61 F F F 15 300-319 32 32 32 58 58 58 F F F 16 320-339 33 33 33 59 59 59 F F F 17 340-359 30 30 30 52 52 52 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 34 34 34 61 61 61 F F F 20 400-419 22 22 22 38 38 38 F F F 21 420-439 33 33 33 59 59 59 F F F 22 440-459 32 32 32 56 56 56 F F F 23 460-479 34 34 34 61 61 61 F F F 24 480-499 30 30 30 52 52 52 F F F 25 500-519 30 30 30 52 52 52 F F F 26 520-539 34 34 34 61 61 61 F F F 27 540-559 35 35 35 62 62 62 F F F 28 560-579 32 32 32 54 54 54 F F F 29 580-599 32 32 32 54 54 54 F F F Max gen: 35, Max flush: 62 Begin loop 18 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:22.994436217Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:22.99473395Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:22.994770988Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:22.994804536Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326} {"msg":"Crucible 9051ba21-6d58-4165-92d2-c5d68c16cd19 has session id: f8515e87-feb6-476c-84ab-d2203d602d7a","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:22.994965277Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"Upstairs opts: Upstairs UUID: 9051ba21-6d58-4165-92d2-c5d68c16cd19, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:22.995041753Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"Crucible stats registered with UUID: 9051ba21-6d58-4165-92d2-c5d68c16cd19","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:22.995065222Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.025868997Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326} {"msg":"9051ba21-6d58-4165-92d2-c5d68c16cd19 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.025899825Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.025918424Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"client":"0","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.025946963Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"client":"0","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.025964902Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"client":"1","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.025982091Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"client":"1","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.02600324Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"client":"2","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.026034078Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"client":"2","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.026065206Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.026096664Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.026128302Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"ds_connection connected from Ok(127.0.0.1:65184)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.026331621Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"ds_connection connected from Ok(127.0.0.1:37215)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.026372079Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"ds_connection connected from Ok(127.0.0.1:62288)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.026404537Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.02705528Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"client":"1","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.027090328Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"client":"1","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.027109967Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"client":"2","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.027128406Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"client":"2","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.027142055Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"client":"0","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.027164314Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"client":"0","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.027748301Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"client":"1","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.027776789Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"client":"2","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.027791978Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"client":"0","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.028316069Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"client":"1","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.028343997Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"client":"1","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.028359236Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.028439282Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"client":"0","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.02847191Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"client":"0","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.028497978Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.028511787Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"client":"2","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.028525527Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"client":"2","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"[0]R flush_numbers[0..12]: [53, 60, 55, 53, 57, 61, 61, 61, 48, 62, 62, 57]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.028539386Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"":"downstairs","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"[0]R generation[0..12]: [31, 34, 32, 31, 32, 34, 34, 34, 28, 35, 35, 32]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.028553325Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"":"downstairs","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.028567674Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"":"downstairs","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"[1]R flush_numbers[0..12]: [53, 60, 55, 53, 57, 61, 61, 61, 48, 62, 62, 57]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.028581803Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"":"downstairs","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"[1]R generation[0..12]: [31, 34, 32, 31, 32, 34, 34, 34, 28, 35, 35, 32]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.028595443Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"":"downstairs","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.028609482Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"":"downstairs","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"[2]R flush_numbers[0..12]: [53, 60, 55, 53, 57, 61, 61, 61, 48, 62, 62, 57]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.028623051Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"":"downstairs","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"[2]R generation[0..12]: [31, 34, 32, 31, 32, 34, 34, 34, 28, 35, 35, 32]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.02863661Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"":"downstairs","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.028650659Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"":"downstairs","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.028671908Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"":"downstairs","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"Generation requested: 36 >= found:36"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.028700597Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"":"downstairs","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"Next flush: 63","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.028728625Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"":"downstairs","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.028744254Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"":"downstairs","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.028757693Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.028769833Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"client":"0","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.028782982Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"client":"1","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.028796161Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"client":"2","":"downstairs","session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.02880927Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.02882181Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"9051ba21-6d58-4165-92d2-c5d68c16cd19 is now active with session: f8515e87-feb6-476c-84ab-d2203d602d7a","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.028834789Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.028846868Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326,"session_id":"f8515e87-feb6-476c-84ab-d2203d602d7a"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:23.028859108Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 46 len 36864 data: 2 2 4 5 5 4 6 6 5 02/30 Read block 442 len 16384 03/30 Write block 179 len 20480 data: 3 4 4 5 6 04/30 Read block 260 len 24576 05/30 Read block 170 len 8192 06/30 Flush 07/30 Write block 216 len 32768 data: 4 5 4 4 3 3 4 4 08/30 Read block 391 len 36864 09/30 Write block 135 len 28672 data: 3 3 3 3 3 4 4 10/30 Read block 456 len 8192 11/30 Read block 338 len 32768 12/30 Write block 76 len 4096 data: 3 13/30 Read block 76 len 4096 14/30 Read block 16 len 8192 15/30 Read block 195 len 20480 16/30 Read block 229 len 40960 17/30 Read block 153 len 16384 18/30 Read block 86 len 16384 19/30 Read block 509 len 40960 20/30 Read block 535 len 20480 21/30 Read block 514 len 40960 22/30 Read block 487 len 20480 23/30 Flush 24/30 Flush 25/30 Write block 340 len 40960 data: 4 3 4 4 4 4 5 4 4 3 26/30 Read block 547 len 36864 27/30 Write block 401 len 24576 data: 6 5 4 4 3 5 28/30 Read block 356 len 36864 29/30 Write block 412 len 40960 data: 3 3 3 3 3 2 3 2 2 2 30/30 Write block 347 len 28672 data: 5 5 4 3 5 5 5 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 10 Done Done Done false 26 Acked 1025 Read 9 Done Done Done false 27 Acked 1026 Write 6 Done Done Done false 28 Acked 1027 Read 9 Done Done Done false 29 Acked 1028 Write 10 Sent Sent Sent false 30 Acked 1029 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:29:23.054623639Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1326} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 53 53 53 F F F 1 020-039 34 34 34 60 60 60 F F F 2 040-059 36 36 36 63 63 63 F F F 3 060-079 36 36 36 64 64 64 F F F 4 080-099 32 32 32 57 57 57 F F F 5 100-119 34 34 34 61 61 61 F F F 6 120-139 36 36 36 64 64 64 F F F 7 140-159 36 36 36 64 64 64 F F F 8 160-179 36 36 36 63 63 63 F F F 9 180-199 36 36 36 63 63 63 F F F 10 200-219 36 36 36 64 64 64 F F F 11 220-239 36 36 36 64 64 64 F F F 12 240-259 32 32 32 54 54 54 F F F 13 260-279 32 32 32 57 57 57 F F F 14 280-299 34 34 34 61 61 61 F F F 15 300-319 32 32 32 58 58 58 F F F 16 320-339 33 33 33 59 59 59 F F F 17 340-359 30 30 30 52 52 52 T T T 18 360-379 24 24 24 42 42 42 F F F 19 380-399 34 34 34 61 61 61 F F F 20 400-419 22 22 22 38 38 38 T T T 21 420-439 33 33 33 59 59 59 T T T 22 440-459 32 32 32 56 56 56 F F F 23 460-479 34 34 34 61 61 61 F F F 24 480-499 30 30 30 52 52 52 F F F 25 500-519 30 30 30 52 52 52 F F F 26 520-539 34 34 34 61 61 61 F F F 27 540-559 35 35 35 62 62 62 F F F 28 560-579 32 32 32 54 54 54 F F F 29 580-599 32 32 32 54 54 54 F F F Max gen: 36, Max flush: 64 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 53 53 53 F F F 1 020-039 34 34 34 60 60 60 F F F 2 040-059 36 36 36 63 63 63 F F F 3 060-079 36 36 36 64 64 64 F F F 4 080-099 32 32 32 57 57 57 F F F 5 100-119 34 34 34 61 61 61 F F F 6 120-139 36 36 36 64 64 64 F F F 7 140-159 36 36 36 64 64 64 F F F 8 160-179 36 36 36 63 63 63 F F F 9 180-199 36 36 36 63 63 63 F F F 10 200-219 36 36 36 64 64 64 F F F 11 220-239 36 36 36 64 64 64 F F F 12 240-259 32 32 32 54 54 54 F F F 13 260-279 32 32 32 57 57 57 F F F 14 280-299 34 34 34 61 61 61 F F F 15 300-319 32 32 32 58 58 58 F F F 16 320-339 33 33 33 59 59 59 F F F 17 340-359 37 37 37 65 65 65 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 34 34 34 61 61 61 F F F 20 400-419 37 37 37 65 65 65 F F F 21 420-439 37 37 37 65 65 65 F F F 22 440-459 32 32 32 56 56 56 F F F 23 460-479 34 34 34 61 61 61 F F F 24 480-499 30 30 30 52 52 52 F F F 25 500-519 30 30 30 52 52 52 F F F 26 520-539 34 34 34 61 61 61 F F F 27 540-559 35 35 35 62 62 62 F F F 28 560-579 32 32 32 54 54 54 F F F 29 580-599 32 32 32 54 54 54 F F F Max gen: 37, Max flush: 65 Begin loop 19 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.407982049Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.408246083Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.408265833Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.408281332Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334} {"msg":"Crucible ad889c94-3eab-4e89-a057-69613f3e2554 has session id: c44de00f-56e8-410e-ad4b-a4d73d0a409c","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.408425424Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"Upstairs opts: Upstairs UUID: ad889c94-3eab-4e89-a057-69613f3e2554, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.408471371Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"Crucible stats registered with UUID: ad889c94-3eab-4e89-a057-69613f3e2554","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.40849522Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.440102959Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334} {"msg":"ad889c94-3eab-4e89-a057-69613f3e2554 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.440138667Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.440161026Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"client":"0","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.440205584Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"client":"0","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.440248161Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"client":"1","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.440284289Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"client":"1","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.440318447Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"client":"2","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.440343086Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"client":"2","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.440374404Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.440407412Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"":"io task","client":"0","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.44044071Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"":"io task","client":"1","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"ds_connection connected from Ok(127.0.0.1:45182)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.440473998Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"":"io task","client":"0","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"ds_connection connected from Ok(127.0.0.1:42087)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.440597041Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"":"io task","client":"1","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"ds_connection connected from Ok(127.0.0.1:51307)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.440635899Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.441373718Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"client":"2","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.441401476Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"client":"2","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.441421285Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"client":"0","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.441435364Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"client":"0","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.441449053Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"client":"1","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.441462212Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"client":"1","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.442084517Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"client":"1","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.442112625Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"client":"2","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.442127915Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"client":"0","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.442680064Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"client":"0","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.442718641Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"client":"0","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.4427467Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.442860083Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"client":"1","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.442888422Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"client":"1","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.442904231Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.44291757Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"client":"2","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.442931329Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"client":"2","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"[0]R flush_numbers[0..12]: [53, 60, 63, 64, 57, 61, 64, 64, 63, 63, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.442945218Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"":"downstairs","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"[0]R generation[0..12]: [31, 34, 36, 36, 32, 34, 36, 36, 36, 36, 36, 36]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.442959228Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"":"downstairs","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.442973767Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"":"downstairs","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"[1]R flush_numbers[0..12]: [53, 60, 63, 64, 57, 61, 64, 64, 63, 63, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.442987386Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"":"downstairs","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"[1]R generation[0..12]: [31, 34, 36, 36, 32, 34, 36, 36, 36, 36, 36, 36]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.443020174Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"":"downstairs","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.443036013Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"":"downstairs","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"[2]R flush_numbers[0..12]: [53, 60, 63, 64, 57, 61, 64, 64, 63, 63, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.443050603Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"":"downstairs","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"[2]R generation[0..12]: [31, 34, 36, 36, 32, 34, 36, 36, 36, 36, 36, 36]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.443064222Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"":"downstairs","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.443078321Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"":"downstairs","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-26T14:29:27.44309141Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"":"downstairs","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.443124368Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"":"downstairs","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"Next flush: 66","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.443137447Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"":"downstairs","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.443150347Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"":"downstairs","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.443163486Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.443175645Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"client":"0","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.443188845Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"client":"1","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.443201954Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"client":"2","":"downstairs","session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.443214963Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.443229302Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"ad889c94-3eab-4e89-a057-69613f3e2554 is now active with session: c44de00f-56e8-410e-ad4b-a4d73d0a409c","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.443242292Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.443254201Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334,"session_id":"c44de00f-56e8-410e-ad4b-a4d73d0a409c"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:27.44326638Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 303 len 40960 data: 4 4 5 5 5 5 5 5 5 5 02/30 Read block 291 len 32768 03/30 Write block 504 len 20480 data: 5 5 5 5 4 04/30 Read block 86 len 40960 05/30 Read block 568 len 16384 06/30 Write block 542 len 36864 data: 2 2 3 4 4 3 4 3 4 07/30 Read block 547 len 40960 08/30 Read block 235 len 20480 09/30 Read block 352 len 36864 10/30 Read block 46 len 20480 11/30 Flush 12/30 Write block 301 len 36864 data: 2 3 5 5 6 6 6 6 6 13/30 Write block 50 len 16384 data: 6 5 7 7 14/30 Read block 330 len 12288 15/30 Write block 540 len 8192 data: 3 3 16/30 Write block 81 len 8192 data: 3 3 17/30 Write block 298 len 24576 data: 5 4 4 3 4 6 18/30 Write block 336 len 8192 data: 3 2 19/30 Write block 573 len 20480 data: 4 4 4 4 4 20/30 Flush 21/30 Write block 302 len 32768 data: 5 7 6 7 7 7 7 7 22/30 Write block 410 len 32768 data: 2 2 4 4 4 4 4 3 23/30 Write block 576 len 24576 data: 5 5 4 3 3 3 24/30 Write block 594 len 8192 data: 4 3 25/30 Write block 65 len 12288 data: 6 7 8 26/30 Read block 281 len 16384 27/30 Read block 71 len 36864 28/30 Write block 305 len 4096 data: 8 29/30 Write block 111 len 8192 data: 4 5 30/30 Read block 51 len 32768 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 8 Done Done Done false 22 Acked 1021 Write 8 Done Done Done false 23 Acked 1022 Write 6 Done Done Done false 24 Acked 1023 Write 2 Done Done Done false 25 Acked 1024 Write 3 Done Done Done false 26 Acked 1025 Read 4 Done Done Done false 27 Acked 1026 Read 9 Done Done Done false 28 Acked 1027 Write 1 Done Done Done false 29 Acked 1028 Write 2 Done Done Sent false 30 Acked 1029 Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 10 9 8 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:29:27.46657544Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1334} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 53 53 53 F F F 1 020-039 34 34 34 60 60 60 F F F 2 040-059 38 38 38 67 67 67 F F F 3 060-079 36 36 36 64 64 64 T T T 4 080-099 38 38 38 67 67 67 F F F 5 100-119 34 34 34 61 61 61 T T T 6 120-139 36 36 36 64 64 64 F F F 7 140-159 36 36 36 64 64 64 F F F 8 160-179 36 36 36 63 63 63 F F F 9 180-199 36 36 36 63 63 63 F F F 10 200-219 36 36 36 64 64 64 F F F 11 220-239 36 36 36 64 64 64 F F F 12 240-259 32 32 32 54 54 54 F F F 13 260-279 32 32 32 57 57 57 F F F 14 280-299 38 38 38 67 67 67 F F F 15 300-319 38 38 38 67 67 67 T T T 16 320-339 38 38 38 67 67 67 F F F 17 340-359 37 37 37 65 65 65 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 34 34 34 61 61 61 F F F 20 400-419 37 37 37 65 65 65 T T T 21 420-439 37 37 37 65 65 65 F F F 22 440-459 32 32 32 56 56 56 F F F 23 460-479 34 34 34 61 61 61 F F F 24 480-499 30 30 30 52 52 52 F F F 25 500-519 38 38 38 66 66 66 F F F 26 520-539 34 34 34 61 61 61 F F F 27 540-559 38 38 38 67 67 67 F F F 28 560-579 38 38 38 67 67 67 T T T 29 580-599 32 32 32 54 54 54 T T T Max gen: 38, Max flush: 67 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 53 53 53 F F F 1 020-039 34 34 34 60 60 60 F F F 2 040-059 38 38 38 67 67 67 F F F 3 060-079 39 39 39 68 68 68 F F F 4 080-099 38 38 38 67 67 67 F F F 5 100-119 39 39 39 68 68 68 F F F 6 120-139 36 36 36 64 64 64 F F F 7 140-159 36 36 36 64 64 64 F F F 8 160-179 36 36 36 63 63 63 F F F 9 180-199 36 36 36 63 63 63 F F F 10 200-219 36 36 36 64 64 64 F F F 11 220-239 36 36 36 64 64 64 F F F 12 240-259 32 32 32 54 54 54 F F F 13 260-279 32 32 32 57 57 57 F F F 14 280-299 38 38 38 67 67 67 F F F 15 300-319 39 39 39 68 68 68 F F F 16 320-339 38 38 38 67 67 67 F F F 17 340-359 37 37 37 65 65 65 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 34 34 34 61 61 61 F F F 20 400-419 39 39 39 68 68 68 F F F 21 420-439 37 37 37 65 65 65 F F F 22 440-459 32 32 32 56 56 56 F F F 23 460-479 34 34 34 61 61 61 F F F 24 480-499 30 30 30 52 52 52 F F F 25 500-519 38 38 38 66 66 66 F F F 26 520-539 34 34 34 61 61 61 F F F 27 540-559 38 38 38 67 67 67 F F F 28 560-579 39 39 39 68 68 68 F F F 29 580-599 39 39 39 68 68 68 F F F Max gen: 39, Max flush: 68 Begin loop 20 Downstairs to restart: 2 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.828988978Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.829223764Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.829245083Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.829265522Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342} {"msg":"Crucible d8c1d654-ae1b-4d59-b4fa-cabad94056cd has session id: cc8806ad-9789-46f5-932e-424deea193e1","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.829433603Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"Upstairs opts: Upstairs UUID: d8c1d654-ae1b-4d59-b4fa-cabad94056cd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.8294835Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"Crucible stats registered with UUID: d8c1d654-ae1b-4d59-b4fa-cabad94056cd","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.829503699Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.860277276Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342} {"msg":"d8c1d654-ae1b-4d59-b4fa-cabad94056cd active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.860305504Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.860322193Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"client":"0","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.860348052Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"client":"0","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.860365281Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"client":"1","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.86038215Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"client":"1","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.860398989Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"client":"2","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.860491164Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"client":"2","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.860524382Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"":"io task","client":"2","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.86054906Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"":"io task","client":"0","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.860567849Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:50834)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.860586048Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"":"io task","client":"2","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:48760)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.860819645Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"":"io task","client":"0","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:42938)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.860850023Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.861352405Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"client":"1","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.861379743Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"client":"1","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.861394593Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"client":"2","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.861408232Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"client":"2","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.861421701Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"client":"0","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.86143497Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"client":"0","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.862025677Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"client":"1","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.862165769Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"client":"0","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.862200527Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"client":"2","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.862543248Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"client":"1","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.862572666Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"client":"1","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.862592115Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.862698359Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"client":"2","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.862722307Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"client":"2","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.862739316Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.862767275Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"client":"0","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.862784714Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"client":"0","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"[0]R flush_numbers[0..12]: [53, 60, 67, 68, 67, 68, 64, 64, 63, 63, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.862815232Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"":"downstairs","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"[0]R generation[0..12]: [31, 34, 38, 39, 38, 39, 36, 36, 36, 36, 36, 36]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.862832931Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"":"downstairs","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.86285055Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"":"downstairs","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"[1]R flush_numbers[0..12]: [53, 60, 67, 68, 67, 68, 64, 64, 63, 63, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.862867669Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"":"downstairs","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"[1]R generation[0..12]: [31, 34, 38, 39, 38, 39, 36, 36, 36, 36, 36, 36]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.862885058Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"":"downstairs","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.862903307Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"":"downstairs","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"[2]R flush_numbers[0..12]: [53, 60, 67, 68, 67, 68, 64, 64, 63, 63, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.862920786Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"":"downstairs","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"[2]R generation[0..12]: [31, 34, 38, 39, 38, 39, 36, 36, 36, 36, 36, 36]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.862937915Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"":"downstairs","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.862960234Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"":"downstairs","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.862977043Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"":"downstairs","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.862993352Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"":"downstairs","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 69","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.86301785Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"":"downstairs","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.863047849Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"":"downstairs","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.863080197Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.863109625Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"client":"0","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.863142594Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"client":"1","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.863175522Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"client":"2","":"downstairs","session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.8632083Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.863244678Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"d8c1d654-ae1b-4d59-b4fa-cabad94056cd is now active with session: cc8806ad-9789-46f5-932e-424deea193e1","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.863278396Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.863306374Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342,"session_id":"cc8806ad-9789-46f5-932e-424deea193e1"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:31.863322883Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 179 len 8192 data: 4 5 02/30 Read block 263 len 4096 03/30 Read block 45 len 8192 04/30 Write block 292 len 20480 data: 5 5 6 7 6 05/30 Write block 478 len 32768 data: 3 5 5 5 4 4 3 4 06/30 Read block 247 len 8192 07/30 Read block 531 len 8192 08/30 Read block 533 len 20480 09/30 Read block 244 len 24576 10/30 Read block 308 len 4096 11/30 Read block 31 len 24576 12/30 Read block 95 len 4096 13/30 Write block 507 len 24576 data: 6 5 4 4 4 4 14/30 Read block 597 len 8192 15/30 Read block 66 len 4096 16/30 Write block 101 len 20480 data: 3 3 3 3 3 17/30 Read block 444 len 40960 18/30 Read block 295 len 4096 19/30 Read block 336 len 4096 20/30 Write block 86 len 24576 data: 4 4 3 3 3 3 21/30 Read block 224 len 24576 22/30 Flush 23/30 Write block 54 len 24576 data: 6 7 7 9 7 8 24/30 Write block 441 len 36864 data: 3 2 3 4 4 5 4 3 4 25/30 Read block 370 len 24576 26/30 Read block 491 len 36864 27/30 Read block 544 len 8192 28/30 Write block 574 len 16384 data: 5 5 6 6 29/30 Write block 8 len 20480 data: 3 3 4 4 5 30/30 Read block 579 len 24576 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 6 Done Done Done false 24 Acked 1023 Write 9 Done Done Done false 25 Acked 1024 Read 6 Done Done Done false 26 Acked 1025 Read 9 Done Done Done false 27 Acked 1026 Read 2 Done Done Done false 28 Acked 1027 Write 4 Done Done Done false 29 Acked 1028 Write 5 Done Sent Sent false 30 Acked 1029 Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 8 6 6 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:29:31.884483555Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1342} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 53 53 53 T T T 1 020-039 34 34 34 60 60 60 F F F 2 040-059 38 38 38 67 67 67 T T T 3 060-079 39 39 39 68 68 68 F F F 4 080-099 40 40 40 69 69 69 F F F 5 100-119 40 40 40 69 69 69 F F F 6 120-139 36 36 36 64 64 64 F F F 7 140-159 36 36 36 64 64 64 F F F 8 160-179 40 40 40 69 69 69 F F F 9 180-199 40 40 40 69 69 69 F F F 10 200-219 36 36 36 64 64 64 F F F 11 220-239 36 36 36 64 64 64 F F F 12 240-259 32 32 32 54 54 54 F F F 13 260-279 32 32 32 57 57 57 F F F 14 280-299 40 40 40 69 69 69 F F F 15 300-319 39 39 39 68 68 68 F F F 16 320-339 38 38 38 67 67 67 F F F 17 340-359 37 37 37 65 65 65 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 34 34 34 61 61 61 F F F 20 400-419 39 39 39 68 68 68 F F F 21 420-439 37 37 37 65 65 65 F F F 22 440-459 32 32 32 56 56 56 T T T 23 460-479 40 40 40 69 69 69 F F F 24 480-499 40 40 40 69 69 69 F F F 25 500-519 40 40 40 69 69 69 F F F 26 520-539 34 34 34 61 61 61 F F F 27 540-559 38 38 38 67 67 67 F F F 28 560-579 39 39 39 68 68 68 T T T 29 580-599 39 39 39 68 68 68 F F F Max gen: 40, Max flush: 69 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 70 70 70 F F F 1 020-039 34 34 34 60 60 60 F F F 2 040-059 41 41 41 70 70 70 F F F 3 060-079 39 39 39 68 68 68 F F F 4 080-099 40 40 40 69 69 69 F F F 5 100-119 40 40 40 69 69 69 F F F 6 120-139 36 36 36 64 64 64 F F F 7 140-159 36 36 36 64 64 64 F F F 8 160-179 40 40 40 69 69 69 F F F 9 180-199 40 40 40 69 69 69 F F F 10 200-219 36 36 36 64 64 64 F F F 11 220-239 36 36 36 64 64 64 F F F 12 240-259 32 32 32 54 54 54 F F F 13 260-279 32 32 32 57 57 57 F F F 14 280-299 40 40 40 69 69 69 F F F 15 300-319 39 39 39 68 68 68 F F F 16 320-339 38 38 38 67 67 67 F F F 17 340-359 37 37 37 65 65 65 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 34 34 34 61 61 61 F F F 20 400-419 39 39 39 68 68 68 F F F 21 420-439 37 37 37 65 65 65 F F F 22 440-459 41 41 41 70 70 70 F F F 23 460-479 40 40 40 69 69 69 F F F 24 480-499 40 40 40 69 69 69 F F F 25 500-519 40 40 40 69 69 69 F F F 26 520-539 34 34 34 61 61 61 F F F 27 540-559 38 38 38 67 67 67 F F F 28 560-579 41 41 41 70 70 70 F F F 29 580-599 39 39 39 68 68 68 F F F Max gen: 41, Max flush: 70 Begin loop 21 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.261624428Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.261919232Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.26194863Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.261980558Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350} {"msg":"Crucible e27e8725-5c5a-41c5-a44e-88d2066759bc has session id: def016e8-704b-46c1-a24c-1c11fc580122","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.262070563Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"Upstairs opts: Upstairs UUID: e27e8725-5c5a-41c5-a44e-88d2066759bc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.262113801Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"Crucible stats registered with UUID: e27e8725-5c5a-41c5-a44e-88d2066759bc","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.262140749Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.295298331Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350} {"msg":"e27e8725-5c5a-41c5-a44e-88d2066759bc active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.2953296Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.295348268Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"client":"0","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.295381526Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"client":"0","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.295413745Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"client":"1","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.295445543Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"client":"1","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.295479521Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"client":"2","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.295513209Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"client":"2","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.295549137Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.295611973Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"":"io task","client":"0","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.295642062Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"ds_connection connected from Ok(127.0.0.1:36600)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.29566722Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"":"io task","client":"0","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"ds_connection connected from Ok(127.0.0.1:53869)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.295693839Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"ds_connection connected from Ok(127.0.0.1:57071)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.295721877Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.296328463Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"client":"1","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.296363211Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"client":"1","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.2963828Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"client":"2","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.296442386Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"client":"2","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.296469945Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"client":"0","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.296493343Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"client":"0","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.296986456Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"client":"1","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.297027603Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"client":"0","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.297059191Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"client":"2","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.297651548Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"client":"1","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.297678816Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"client":"1","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.297693815Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.297706575Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"client":"0","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.297720214Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"client":"0","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.297733443Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.297745503Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"client":"2","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.297762172Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"client":"2","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"[0]R flush_numbers[0..12]: [70, 60, 70, 68, 69, 69, 64, 64, 69, 69, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.297776711Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"":"downstairs","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"[0]R generation[0..12]: [41, 34, 41, 39, 40, 40, 36, 36, 40, 40, 36, 36]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.2977909Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"":"downstairs","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.297806339Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"":"downstairs","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"[1]R flush_numbers[0..12]: [70, 60, 70, 68, 69, 69, 64, 64, 69, 69, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.297820059Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"":"downstairs","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"[1]R generation[0..12]: [41, 34, 41, 39, 40, 40, 36, 36, 40, 40, 36, 36]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.297833588Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"":"downstairs","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.297847587Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"":"downstairs","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"[2]R flush_numbers[0..12]: [70, 60, 70, 68, 69, 69, 64, 64, 69, 69, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.297861036Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"":"downstairs","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"[2]R generation[0..12]: [41, 34, 41, 39, 40, 40, 36, 36, 40, 40, 36, 36]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.297874465Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"":"downstairs","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.297888624Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"":"downstairs","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"Max found gen is 42","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2024-09-26T14:29:36.297904924Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"":"downstairs","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.297918333Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"":"downstairs","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"Next flush: 71","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.297931042Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"":"downstairs","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.297943872Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"":"downstairs","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.297959471Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.29797172Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"client":"0","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.297985099Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"client":"1","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.297998058Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"client":"2","":"downstairs","session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.298011248Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.298023437Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"e27e8725-5c5a-41c5-a44e-88d2066759bc is now active with session: def016e8-704b-46c1-a24c-1c11fc580122","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.298036156Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.298048026Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350,"session_id":"def016e8-704b-46c1-a24c-1c11fc580122"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:36.298060145Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 353 len 24576 data: 6 5 6 4 4 5 02/30 Write block 414 len 16384 data: 5 5 5 4 03/30 Write block 419 len 20480 data: 3 3 3 3 4 04/30 Write block 107 len 4096 data: 3 05/30 Read block 291 len 28672 06/30 Read block 107 len 4096 07/30 Write block 303 len 24576 data: 8 7 9 8 8 8 08/30 Flush 09/30 Write block 12 len 32768 data: 6 5 4 4 4 4 4 3 10/30 Write block 78 len 24576 data: 3 3 3 4 4 3 11/30 Write block 108 len 40960 data: 4 3 3 5 6 6 6 5 3 3 12/30 Read block 118 len 28672 13/30 Read block 32 len 32768 14/30 Read block 20 len 16384 15/30 Write block 14 len 24576 data: 5 5 5 5 5 4 16/30 Read block 356 len 36864 17/30 Read block 492 len 24576 18/30 Write block 7 len 8192 data: 3 4 19/30 Read block 541 len 24576 20/30 Write block 116 len 24576 data: 4 4 5 5 5 6 21/30 Read block 533 len 4096 22/30 Read block 195 len 24576 23/30 Read block 412 len 16384 24/30 Flush 25/30 Write block 288 len 36864 data: 5 6 6 5 6 6 7 8 7 26/30 Read block 260 len 8192 27/30 Read block 25 len 16384 28/30 Write block 3 len 8192 data: 3 3 29/30 Read block 33 len 32768 30/30 Read block 183 len 8192 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 9 Done Done Done false 26 Acked 1025 Read 2 Done Done Done false 27 Acked 1026 Read 4 Done Done Done false 28 Acked 1027 Write 2 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:29:36.322002959Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1350} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 72 72 72 T T T 1 020-039 34 34 34 60 60 60 F F F 2 040-059 41 41 41 70 70 70 F F F 3 060-079 42 42 42 72 72 72 F F F 4 080-099 42 42 42 72 72 72 F F F 5 100-119 42 42 42 72 72 72 F F F 6 120-139 42 42 42 72 72 72 F F F 7 140-159 36 36 36 64 64 64 F F F 8 160-179 40 40 40 69 69 69 F F F 9 180-199 40 40 40 69 69 69 F F F 10 200-219 36 36 36 64 64 64 F F F 11 220-239 36 36 36 64 64 64 F F F 12 240-259 32 32 32 54 54 54 F F F 13 260-279 32 32 32 57 57 57 F F F 14 280-299 40 40 40 69 69 69 T T T 15 300-319 42 42 42 71 71 71 F F F 16 320-339 38 38 38 67 67 67 F F F 17 340-359 42 42 42 71 71 71 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 34 34 34 61 61 61 F F F 20 400-419 42 42 42 71 71 71 F F F 21 420-439 42 42 42 71 71 71 F F F 22 440-459 41 41 41 70 70 70 F F F 23 460-479 40 40 40 69 69 69 F F F 24 480-499 40 40 40 69 69 69 F F F 25 500-519 40 40 40 69 69 69 F F F 26 520-539 34 34 34 61 61 61 F F F 27 540-559 38 38 38 67 67 67 F F F 28 560-579 41 41 41 70 70 70 F F F 29 580-599 39 39 39 68 68 68 F F F Max gen: 42, Max flush: 72 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 73 73 73 F F F 1 020-039 34 34 34 60 60 60 F F F 2 040-059 41 41 41 70 70 70 F F F 3 060-079 42 42 42 72 72 72 F F F 4 080-099 42 42 42 72 72 72 F F F 5 100-119 42 42 42 72 72 72 F F F 6 120-139 42 42 42 72 72 72 F F F 7 140-159 36 36 36 64 64 64 F F F 8 160-179 40 40 40 69 69 69 F F F 9 180-199 40 40 40 69 69 69 F F F 10 200-219 36 36 36 64 64 64 F F F 11 220-239 36 36 36 64 64 64 F F F 12 240-259 32 32 32 54 54 54 F F F 13 260-279 32 32 32 57 57 57 F F F 14 280-299 43 43 43 73 73 73 F F F 15 300-319 42 42 42 71 71 71 F F F 16 320-339 38 38 38 67 67 67 F F F 17 340-359 42 42 42 71 71 71 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 34 34 34 61 61 61 F F F 20 400-419 42 42 42 71 71 71 F F F 21 420-439 42 42 42 71 71 71 F F F 22 440-459 41 41 41 70 70 70 F F F 23 460-479 40 40 40 69 69 69 F F F 24 480-499 40 40 40 69 69 69 F F F 25 500-519 40 40 40 69 69 69 F F F 26 520-539 34 34 34 61 61 61 F F F 27 540-559 38 38 38 67 67 67 F F F 28 560-579 41 41 41 70 70 70 F F F 29 580-599 39 39 39 68 68 68 F F F Max gen: 43, Max flush: 73 Begin loop 22 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.693044358Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.693321372Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.693339391Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.693355741Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358} {"msg":"Crucible 547c007e-cb53-4c30-a150-bc267b5632fa has session id: 9a8c69d7-4356-42c0-970f-bed796652151","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.69353186Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"Upstairs opts: Upstairs UUID: 547c007e-cb53-4c30-a150-bc267b5632fa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.693583928Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"Crucible stats registered with UUID: 547c007e-cb53-4c30-a150-bc267b5632fa","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.693608286Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.724678127Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358} {"msg":"547c007e-cb53-4c30-a150-bc267b5632fa active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.724714694Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.724737753Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"client":"0","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.724781831Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"client":"0","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.724819928Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"client":"1","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.724855377Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"client":"1","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.724887605Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"client":"2","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.724919393Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"client":"2","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.72496636Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"":"io task","client":"2","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.724997649Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"":"io task","client":"0","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.725137091Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"ds_connection connected from Ok(127.0.0.1:38623)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.725185748Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"":"io task","client":"2","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"ds_connection connected from Ok(127.0.0.1:55794)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.725220006Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"ds_connection connected from Ok(127.0.0.1:62208)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.725254754Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"":"io task","client":"0","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.725815962Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"client":"1","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.7258502Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"client":"1","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.725874479Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"client":"0","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.725900837Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"client":"0","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.725927046Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"client":"2","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.725955064Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"client":"2","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.726511603Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"client":"1","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.726539531Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"client":"2","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.72655475Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"client":"0","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.727139617Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"client":"1","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.727177605Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"client":"1","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.727205664Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.727229442Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"client":"2","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.727252151Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"client":"2","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.72727331Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.727294898Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"client":"0","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.727318877Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"client":"0","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"[0]R flush_numbers[0..12]: [73, 60, 70, 72, 72, 72, 72, 64, 69, 69, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.727343786Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"":"downstairs","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"[0]R generation[0..12]: [43, 34, 41, 42, 42, 42, 42, 36, 40, 40, 36, 36]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.727367144Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"":"downstairs","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.727390583Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"":"downstairs","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"[1]R flush_numbers[0..12]: [73, 60, 70, 72, 72, 72, 72, 64, 69, 69, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.727419201Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"":"downstairs","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"[1]R generation[0..12]: [43, 34, 41, 42, 42, 42, 42, 36, 40, 40, 36, 36]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.72744644Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"":"downstairs","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.727471348Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"":"downstairs","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"[2]R flush_numbers[0..12]: [73, 60, 70, 72, 72, 72, 72, 64, 69, 69, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.727495527Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"":"downstairs","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"[2]R generation[0..12]: [43, 34, 41, 42, 42, 42, 42, 36, 40, 40, 36, 36]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.727522886Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.727555644Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"":"downstairs","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.727582632Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"":"downstairs","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.727609641Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"":"downstairs","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"Next flush: 74","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.727636009Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"":"downstairs","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.727663098Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"":"downstairs","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.727689646Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 1358,"session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.727720884Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"client":"0","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.727744123Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"client":"1","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.727759832Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"client":"2","":"downstairs","session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.727773871Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.72779001Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"547c007e-cb53-4c30-a150-bc267b5632fa is now active with session: 9a8c69d7-4356-42c0-970f-bed796652151","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.72780331Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.727815199Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358,"session_id":"9a8c69d7-4356-42c0-970f-bed796652151"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:40.727827468Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 28 len 24576 data: 6 3 4 4 4 4 02/30 Read block 335 len 40960 03/30 Read block 366 len 20480 04/30 Read block 90 len 40960 05/30 Write block 535 len 24576 data: 4 5 4 5 4 4 06/30 Write block 518 len 40960 data: 4 4 4 4 4 5 5 5 5 4 07/30 Write block 274 len 20480 data: 3 4 4 5 5 08/30 Write block 492 len 28672 data: 3 3 4 4 4 4 5 09/30 Read block 148 len 36864 10/30 Read block 259 len 16384 11/30 Read block 587 len 4096 12/30 Write block 577 len 36864 data: 7 5 4 4 4 4 5 3 3 13/30 Write block 25 len 36864 data: 6 7 7 7 4 5 5 5 5 14/30 Write block 260 len 16384 data: 4 6 7 7 15/30 Write block 79 len 16384 data: 4 4 5 5 16/30 Read block 204 len 40960 17/30 Read block 69 len 32768 18/30 Read block 383 len 36864 19/30 Flush 20/30 Write block 278 len 20480 data: 6 5 5 4 5 21/30 Read block 373 len 40960 22/30 Write block 274 len 32768 data: 4 5 5 6 7 6 6 5 23/30 Read block 71 len 16384 24/30 Read block 586 len 16384 25/30 Write block 367 len 20480 data: 2 2 4 4 4 26/30 Read block 363 len 16384 27/30 Read block 184 len 12288 28/30 Read block 528 len 28672 29/30 Read block 272 len 28672 30/30 Read block 133 len 36864 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 20 Acked 1019 Write 5 Done Done Done false 21 Acked 1020 Read 10 Done Done Done false 22 Acked 1021 Write 8 Done Done Done false 23 Acked 1022 Read 4 Done Done Done false 24 Acked 1023 Read 4 Done Done Done false 25 Acked 1024 Write 5 Done Done Done false 26 Acked 1025 Read 4 Done Done Done false 27 Acked 1026 Read 3 Done Done Done false 28 Acked 1027 Read 7 Done Done Done false 29 Acked 1028 Read 7 Done Done Done false 30 Acked 1029 Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 10 11 10 31 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:29:40.752576567Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1358} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 73 73 73 F F F 1 020-039 44 44 44 74 74 74 F F F 2 040-059 41 41 41 70 70 70 F F F 3 060-079 44 44 44 74 74 74 F F F 4 080-099 44 44 44 74 74 74 F F F 5 100-119 42 42 42 72 72 72 F F F 6 120-139 42 42 42 72 72 72 F F F 7 140-159 36 36 36 64 64 64 F F F 8 160-179 40 40 40 69 69 69 F F F 9 180-199 40 40 40 69 69 69 F F F 10 200-219 36 36 36 64 64 64 F F F 11 220-239 36 36 36 64 64 64 F F F 12 240-259 32 32 32 54 54 54 F F F 13 260-279 44 44 44 74 74 74 T T T 14 280-299 43 43 43 73 73 73 T T T 15 300-319 42 42 42 71 71 71 F F F 16 320-339 38 38 38 67 67 67 F F F 17 340-359 42 42 42 71 71 71 F F F 18 360-379 24 24 24 42 42 42 T T T 19 380-399 34 34 34 61 61 61 F F F 20 400-419 42 42 42 71 71 71 F F F 21 420-439 42 42 42 71 71 71 F F F 22 440-459 41 41 41 70 70 70 F F F 23 460-479 40 40 40 69 69 69 F F F 24 480-499 44 44 44 74 74 74 F F F 25 500-519 44 44 44 74 74 74 F F F 26 520-539 44 44 44 74 74 74 F F F 27 540-559 44 44 44 74 74 74 F F F 28 560-579 44 44 44 74 74 74 F F F 29 580-599 44 44 44 74 74 74 F F F Max gen: 44, Max flush: 74 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 73 73 73 F F F 1 020-039 44 44 44 74 74 74 F F F 2 040-059 41 41 41 70 70 70 F F F 3 060-079 44 44 44 74 74 74 F F F 4 080-099 44 44 44 74 74 74 F F F 5 100-119 42 42 42 72 72 72 F F F 6 120-139 42 42 42 72 72 72 F F F 7 140-159 36 36 36 64 64 64 F F F 8 160-179 40 40 40 69 69 69 F F F 9 180-199 40 40 40 69 69 69 F F F 10 200-219 36 36 36 64 64 64 F F F 11 220-239 36 36 36 64 64 64 F F F 12 240-259 32 32 32 54 54 54 F F F 13 260-279 45 45 45 75 75 75 F F F 14 280-299 45 45 45 75 75 75 F F F 15 300-319 42 42 42 71 71 71 F F F 16 320-339 38 38 38 67 67 67 F F F 17 340-359 42 42 42 71 71 71 F F F 18 360-379 45 45 45 75 75 75 F F F 19 380-399 34 34 34 61 61 61 F F F 20 400-419 42 42 42 71 71 71 F F F 21 420-439 42 42 42 71 71 71 F F F 22 440-459 41 41 41 70 70 70 F F F 23 460-479 40 40 40 69 69 69 F F F 24 480-499 44 44 44 74 74 74 F F F 25 500-519 44 44 44 74 74 74 F F F 26 520-539 44 44 44 74 74 74 F F F 27 540-559 44 44 44 74 74 74 F F F 28 560-579 44 44 44 74 74 74 F F F 29 580-599 44 44 44 74 74 74 F F F Max gen: 45, Max flush: 75 Begin loop 23 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.108922717Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.109284637Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.109324245Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.109358373Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366} {"msg":"Crucible 6dc07432-2088-4e8d-aaa7-861b4c4583fe has session id: 9d22972b-cab1-4088-a336-b5bc33b03a7c","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.109446758Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"Upstairs opts: Upstairs UUID: 6dc07432-2088-4e8d-aaa7-861b4c4583fe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.109501275Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"Crucible stats registered with UUID: 6dc07432-2088-4e8d-aaa7-861b4c4583fe","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.109529063Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.140383696Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366} {"msg":"6dc07432-2088-4e8d-aaa7-861b4c4583fe active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.140415874Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.140434583Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"client":"0","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.140461421Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"client":"0","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.14047927Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"client":"1","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.140499589Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"client":"1","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.140516498Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"client":"2","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.140533027Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"client":"2","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.140589944Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.140642161Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"":"io task","client":"0","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.140678769Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"":"io task","client":"1","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"ds_connection connected from Ok(127.0.0.1:52429)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.140712487Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"ds_connection connected from Ok(127.0.0.1:53247)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.140867668Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"":"io task","client":"0","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"ds_connection connected from Ok(127.0.0.1:55913)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.140909426Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"":"io task","client":"1","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.141491083Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"client":"2","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.141518301Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"client":"2","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.141533501Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"client":"1","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.1415473Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"client":"1","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.141561189Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"client":"0","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.141574758Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"client":"0","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.142170225Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"client":"2","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.142309087Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"client":"1","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.142338265Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"client":"0","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.142715454Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"client":"2","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.142746522Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"client":"2","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.142761801Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.142893744Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"client":"0","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.142921692Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"client":"0","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.142936851Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.14296227Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"client":"1","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.142976729Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"client":"1","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"[0]R flush_numbers[0..12]: [73, 74, 70, 74, 74, 72, 72, 64, 69, 69, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.143049895Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"":"downstairs","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"[0]R generation[0..12]: [43, 44, 41, 44, 44, 42, 42, 36, 40, 40, 36, 36]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.143069514Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"":"downstairs","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.143084073Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"":"downstairs","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"[1]R flush_numbers[0..12]: [73, 74, 70, 74, 74, 72, 72, 64, 69, 69, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.143098612Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"":"downstairs","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"[1]R generation[0..12]: [43, 44, 41, 44, 44, 42, 42, 36, 40, 40, 36, 36]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.143112341Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"":"downstairs","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.1431262Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"":"downstairs","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"[2]R flush_numbers[0..12]: [73, 74, 70, 74, 74, 72, 72, 64, 69, 69, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.14313999Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"":"downstairs","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"[2]R generation[0..12]: [43, 44, 41, 44, 44, 42, 42, 36, 40, 40, 36, 36]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.143153649Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"":"downstairs","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.143171248Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"":"downstairs","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.143184487Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"":"downstairs","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.143197656Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"":"downstairs","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"Next flush: 76","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.143210595Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"":"downstairs","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-26T14:29:45.143223535Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"":"downstairs","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.143236854Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.143248813Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"client":"0","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.143262133Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"client":"1","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.143275272Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"client":"2","":"downstairs","session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.143288471Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.14330137Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"6dc07432-2088-4e8d-aaa7-861b4c4583fe is now active with session: 9d22972b-cab1-4088-a336-b5bc33b03a7c","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.143324139Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.143345048Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366,"session_id":"9d22972b-cab1-4088-a336-b5bc33b03a7c"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:45.143358167Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 43 len 36864 data: 3 2 2 3 3 5 6 7 6 03/30 Flush 04/30 Write block 284 len 12288 data: 7 7 6 05/30 Write block 510 len 20480 data: 5 5 5 3 4 06/30 Read block 213 len 24576 07/30 Read block 225 len 20480 08/30 Flush 09/30 Write block 313 len 20480 data: 6 6 5 5 6 10/30 Read block 416 len 20480 11/30 Write block 479 len 36864 data: 6 6 6 5 5 4 5 5 4 12/30 Write block 245 len 4096 data: 4 13/30 Read block 447 len 28672 14/30 Read block 16 len 4096 15/30 Read block 87 len 16384 16/30 Write block 326 len 20480 data: 3 4 5 4 4 17/30 Flush 18/30 Write block 88 len 28672 data: 4 4 4 4 3 3 2 19/30 Write block 302 len 12288 data: 6 9 8 20/30 Write block 227 len 40960 data: 5 5 3 4 2 2 3 3 3 3 21/30 Flush 22/30 Flush 23/30 Flush 24/30 Write block 311 len 12288 data: 6 6 7 25/30 Read block 275 len 16384 26/30 Read block 318 len 32768 27/30 Write block 71 len 36864 data: 6 5 5 3 2 4 3 4 5 28/30 Read block 515 len 32768 29/30 Read block 586 len 24576 30/30 Write block 275 len 32768 data: 6 6 7 8 7 7 6 6 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 3 Done Done Done false 25 Acked 1024 Read 4 Done Done Done false 26 Acked 1025 Read 8 Done Done Done false 27 Acked 1026 Write 9 Done Done Done false 28 Acked 1027 Read 8 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:29:45.171438507Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1366} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 73 73 73 F F F 1 020-039 44 44 44 74 74 74 F F F 2 040-059 46 46 46 77 77 77 F F F 3 060-079 44 44 44 74 74 74 T T T 4 080-099 46 46 46 80 80 80 F F F 5 100-119 42 42 42 72 72 72 F F F 6 120-139 42 42 42 72 72 72 F F F 7 140-159 36 36 36 64 64 64 F F F 8 160-179 40 40 40 69 69 69 F F F 9 180-199 40 40 40 69 69 69 F F F 10 200-219 36 36 36 64 64 64 F F F 11 220-239 46 46 46 80 80 80 F F F 12 240-259 46 46 46 79 79 79 F F F 13 260-279 45 45 45 75 75 75 T T T 14 280-299 46 46 46 78 78 78 T T T 15 300-319 46 46 46 80 80 80 T T T 16 320-339 46 46 46 79 79 79 F F F 17 340-359 42 42 42 71 71 71 F F F 18 360-379 45 45 45 75 75 75 F F F 19 380-399 34 34 34 61 61 61 F F F 20 400-419 42 42 42 71 71 71 F F F 21 420-439 42 42 42 71 71 71 F F F 22 440-459 41 41 41 70 70 70 F F F 23 460-479 46 46 46 79 79 79 F F F 24 480-499 46 46 46 79 79 79 F F F 25 500-519 46 46 46 78 78 78 F F F 26 520-539 44 44 44 74 74 74 F F F 27 540-559 44 44 44 74 74 74 F F F 28 560-579 44 44 44 74 74 74 F F F 29 580-599 44 44 44 74 74 74 F F F Max gen: 46, Max flush: 80 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 73 73 73 F F F 1 020-039 44 44 44 74 74 74 F F F 2 040-059 46 46 46 77 77 77 F F F 3 060-079 47 47 47 81 81 81 F F F 4 080-099 46 46 46 80 80 80 F F F 5 100-119 42 42 42 72 72 72 F F F 6 120-139 42 42 42 72 72 72 F F F 7 140-159 36 36 36 64 64 64 F F F 8 160-179 40 40 40 69 69 69 F F F 9 180-199 40 40 40 69 69 69 F F F 10 200-219 36 36 36 64 64 64 F F F 11 220-239 46 46 46 80 80 80 F F F 12 240-259 46 46 46 79 79 79 F F F 13 260-279 47 47 47 81 81 81 F F F 14 280-299 47 47 47 81 81 81 F F F 15 300-319 47 47 47 81 81 81 F F F 16 320-339 46 46 46 79 79 79 F F F 17 340-359 42 42 42 71 71 71 F F F 18 360-379 45 45 45 75 75 75 F F F 19 380-399 34 34 34 61 61 61 F F F 20 400-419 42 42 42 71 71 71 F F F 21 420-439 42 42 42 71 71 71 F F F 22 440-459 41 41 41 70 70 70 F F F 23 460-479 46 46 46 79 79 79 F F F 24 480-499 46 46 46 79 79 79 F F F 25 500-519 46 46 46 78 78 78 F F F 26 520-539 44 44 44 74 74 74 F F F 27 540-559 44 44 44 74 74 74 F F F 28 560-579 44 44 44 74 74 74 F F F 29 580-599 44 44 44 74 74 74 F F F Max gen: 47, Max flush: 81 Begin loop 24 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.539109777Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.539376122Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.53940505Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.539435789Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374} {"msg":"Crucible 04cf9c11-f0ff-476b-85cc-4b78e21f5afb has session id: 17deb433-4f5f-4ded-8ba0-756e5b6e0ead","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.539561102Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"Upstairs opts: Upstairs UUID: 04cf9c11-f0ff-476b-85cc-4b78e21f5afb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.539608679Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"Crucible stats registered with UUID: 04cf9c11-f0ff-476b-85cc-4b78e21f5afb","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.539636587Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.571079606Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374} {"msg":"04cf9c11-f0ff-476b-85cc-4b78e21f5afb active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.571109375Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.571126564Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"client":"0","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.571153082Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"client":"0","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.571170611Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"client":"1","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.57118821Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"client":"1","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.571217579Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"client":"2","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.571249277Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"client":"2","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.571314123Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.571351161Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"":"io task","client":"0","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.57137846Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"ds_connection connected from Ok(127.0.0.1:53410)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.571404898Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"ds_connection connected from Ok(127.0.0.1:34852)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.571433976Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"ds_connection connected from Ok(127.0.0.1:40878)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.571585088Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"":"io task","client":"0","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.572219422Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"client":"1","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.57224634Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"client":"1","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.572261119Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"client":"2","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.572275079Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"client":"2","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.572296508Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"client":"0","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.572322546Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"client":"0","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.572928972Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"client":"1","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.573088503Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"client":"2","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.573126721Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"client":"0","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.573537197Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"client":"1","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.573565856Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"client":"1","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.573581655Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.573628342Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"client":"0","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.573656471Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"client":"0","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.57367285Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.573685769Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"client":"2","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.573702588Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"client":"2","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"[0]R flush_numbers[0..12]: [73, 74, 77, 81, 80, 72, 72, 64, 69, 69, 64, 80]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.573716667Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"":"downstairs","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"[0]R generation[0..12]: [43, 44, 46, 47, 46, 42, 42, 36, 40, 40, 36, 46]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.573730626Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"":"downstairs","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.573744926Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"":"downstairs","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"[1]R flush_numbers[0..12]: [73, 74, 77, 81, 80, 72, 72, 64, 69, 69, 64, 80]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.573758635Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"":"downstairs","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"[1]R generation[0..12]: [43, 44, 46, 47, 46, 42, 42, 36, 40, 40, 36, 46]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.573772174Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"":"downstairs","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.573786363Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"":"downstairs","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"[2]R flush_numbers[0..12]: [73, 74, 77, 81, 80, 72, 72, 64, 69, 69, 64, 80]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.573799932Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"":"downstairs","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"[2]R generation[0..12]: [43, 44, 46, 47, 46, 42, 42, 36, 40, 40, 36, 46]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.573813522Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"":"downstairs","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.573827521Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"":"downstairs","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.57384047Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"":"downstairs","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.573853619Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"":"downstairs","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"Next flush: 82","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.573876888Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"":"downstairs","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.573893387Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"":"downstairs","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-09-26T14:29:49.573909816Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.573925295Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"client":"0","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.573939064Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"client":"1","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.573952344Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"client":"2","":"downstairs","session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.573965823Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.573978332Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"04cf9c11-f0ff-476b-85cc-4b78e21f5afb is now active with session: 17deb433-4f5f-4ded-8ba0-756e5b6e0ead","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.573991281Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.574003301Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374,"session_id":"17deb433-4f5f-4ded-8ba0-756e5b6e0ead"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:49.57401566Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 543 len 28672 data: 3 4 5 5 4 5 4 03/30 Flush 04/30 Write block 502 len 4096 data: 8 05/30 Write block 357 len 16384 data: 5 6 4 4 06/30 Read block 417 len 36864 07/30 Write block 569 len 40960 data: 4 4 4 4 5 6 6 7 8 6 08/30 Write block 344 len 40960 data: 5 5 6 6 6 5 4 6 6 7 09/30 Write block 463 len 36864 data: 2 2 3 4 3 2 2 2 3 10/30 Read block 8 len 40960 11/30 Read block 205 len 36864 12/30 Read block 543 len 24576 13/30 Read block 319 len 20480 14/30 Read block 98 len 12288 15/30 Read block 411 len 16384 16/30 Write block 597 len 8192 data: 3 3 17/30 Read block 29 len 16384 18/30 Read block 192 len 36864 19/30 Write block 48 len 4096 data: 6 20/30 Write block 180 len 16384 data: 6 5 6 7 21/30 Flush 22/30 Flush 23/30 Write block 550 len 36864 data: 5 4 3 3 4 4 4 3 3 24/30 Read block 52 len 8192 25/30 Read block 533 len 20480 26/30 Read block 290 len 16384 27/30 Read block 325 len 32768 28/30 Read block 465 len 4096 29/30 Write block 247 len 4096 data: 4 30/30 Read block 82 len 36864 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 9 Done Done Done false 24 Acked 1023 Read 2 Done Done Done false 25 Acked 1024 Read 5 Done Done Done false 26 Acked 1025 Read 4 Done Done Done false 27 Acked 1026 Read 8 Done Done Done false 28 Acked 1027 Read 1 Done Done Done false 29 Acked 1028 Write 1 Done Done Done false 30 Acked 1029 Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 7 8 7 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:29:49.599004335Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1374} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 73 73 73 F F F 1 020-039 44 44 44 74 74 74 F F F 2 040-059 48 48 48 84 84 84 F F F 3 060-079 47 47 47 81 81 81 F F F 4 080-099 46 46 46 80 80 80 F F F 5 100-119 42 42 42 72 72 72 F F F 6 120-139 42 42 42 72 72 72 F F F 7 140-159 36 36 36 64 64 64 F F F 8 160-179 40 40 40 69 69 69 F F F 9 180-199 48 48 48 84 84 84 F F F 10 200-219 36 36 36 64 64 64 F F F 11 220-239 46 46 46 80 80 80 F F F 12 240-259 46 46 46 79 79 79 T T T 13 260-279 47 47 47 81 81 81 F F F 14 280-299 47 47 47 81 81 81 F F F 15 300-319 47 47 47 81 81 81 F F F 16 320-339 46 46 46 79 79 79 F F F 17 340-359 48 48 48 84 84 84 F F F 18 360-379 48 48 48 84 84 84 F F F 19 380-399 34 34 34 61 61 61 F F F 20 400-419 42 42 42 71 71 71 F F F 21 420-439 42 42 42 71 71 71 F F F 22 440-459 41 41 41 70 70 70 F F F 23 460-479 48 48 48 84 84 84 F F F 24 480-499 46 46 46 79 79 79 F F F 25 500-519 48 48 48 84 84 84 F F F 26 520-539 44 44 44 74 74 74 F F F 27 540-559 48 48 48 83 83 83 T T T 28 560-579 48 48 48 84 84 84 F F F 29 580-599 48 48 48 84 84 84 F F F Max gen: 48, Max flush: 84 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 73 73 73 F F F 1 020-039 44 44 44 74 74 74 F F F 2 040-059 48 48 48 84 84 84 F F F 3 060-079 47 47 47 81 81 81 F F F 4 080-099 46 46 46 80 80 80 F F F 5 100-119 42 42 42 72 72 72 F F F 6 120-139 42 42 42 72 72 72 F F F 7 140-159 36 36 36 64 64 64 F F F 8 160-179 40 40 40 69 69 69 F F F 9 180-199 48 48 48 84 84 84 F F F 10 200-219 36 36 36 64 64 64 F F F 11 220-239 46 46 46 80 80 80 F F F 12 240-259 49 49 49 85 85 85 F F F 13 260-279 47 47 47 81 81 81 F F F 14 280-299 47 47 47 81 81 81 F F F 15 300-319 47 47 47 81 81 81 F F F 16 320-339 46 46 46 79 79 79 F F F 17 340-359 48 48 48 84 84 84 F F F 18 360-379 48 48 48 84 84 84 F F F 19 380-399 34 34 34 61 61 61 F F F 20 400-419 42 42 42 71 71 71 F F F 21 420-439 42 42 42 71 71 71 F F F 22 440-459 41 41 41 70 70 70 F F F 23 460-479 48 48 48 84 84 84 F F F 24 480-499 46 46 46 79 79 79 F F F 25 500-519 48 48 48 84 84 84 F F F 26 520-539 44 44 44 74 74 74 F F F 27 540-559 49 49 49 85 85 85 F F F 28 560-579 48 48 48 84 84 84 F F F 29 580-599 48 48 48 84 84 84 F F F Max gen: 49, Max flush: 85 Begin loop 25 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:53.969536233Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:53.969819556Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:53.969847015Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:53.969873013Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382} {"msg":"Crucible 7c5a18ad-1a70-409c-b2b0-3a5cedd62009 has session id: a95fc09f-1534-4a3a-aa18-976c2c9bf439","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:53.969967848Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"Upstairs opts: Upstairs UUID: 7c5a18ad-1a70-409c-b2b0-3a5cedd62009, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:53.970012415Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"Crucible stats registered with UUID: 7c5a18ad-1a70-409c-b2b0-3a5cedd62009","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:53.970036134Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.001324482Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382} {"msg":"7c5a18ad-1a70-409c-b2b0-3a5cedd62009 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.00135535Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.001382889Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"client":"0","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.001412797Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"client":"0","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.001442086Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"client":"1","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.001464804Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"client":"1","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.001486043Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"client":"2","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.001508332Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"client":"2","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.00153103Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.001557589Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.001585087Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"ds_connection connected from Ok(127.0.0.1:39505)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.001613426Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"ds_connection connected from Ok(127.0.0.1:34757)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.001704971Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"ds_connection connected from Ok(127.0.0.1:63015)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.001826424Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.00242645Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"client":"1","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.002454448Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"client":"1","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.002470197Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"client":"2","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.002484987Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"client":"2","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.002566172Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"client":"0","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.002589941Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"client":"0","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.003012297Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"client":"1","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.003037175Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"client":"2","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.003273492Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"client":"0","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.003550426Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"client":"1","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.003573895Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"client":"1","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.003587994Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.00365232Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"client":"2","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.003673139Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"client":"2","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.003689868Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.003824891Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"client":"0","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.003847429Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"client":"0","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"[0]R flush_numbers[0..12]: [73, 74, 84, 81, 80, 72, 72, 64, 69, 84, 64, 80]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.003864818Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"":"downstairs","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"[0]R generation[0..12]: [43, 44, 48, 47, 46, 42, 42, 36, 40, 48, 36, 46]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.003882687Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"":"downstairs","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.003900416Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"":"downstairs","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"[1]R flush_numbers[0..12]: [73, 74, 84, 81, 80, 72, 72, 64, 69, 84, 64, 80]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.003921985Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"":"downstairs","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"[1]R generation[0..12]: [43, 44, 48, 47, 46, 42, 42, 36, 40, 48, 36, 46]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.003939504Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"":"downstairs","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.003957093Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"":"downstairs","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"[2]R flush_numbers[0..12]: [73, 74, 84, 81, 80, 72, 72, 64, 69, 84, 64, 80]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.003974182Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"":"downstairs","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"[2]R generation[0..12]: [43, 44, 48, 47, 46, 42, 42, 36, 40, 48, 36, 46]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.003991231Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"":"downstairs","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.00400864Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"":"downstairs","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.004024959Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"":"downstairs","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.004041348Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"":"downstairs","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"Next flush: 86"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.004060307Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"":"downstairs","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.004086476Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"":"downstairs","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.004105025Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.004120104Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"client":"0","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.004136633Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"client":"1","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.004154492Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"client":"2","":"downstairs","session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.004171091Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msgRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.00419414Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"7c5a18ad-1a70-409c-b2b0-3a5cedd62009 is now active with session: a95fc09f-1534-4a3a-aa18-976c2c9bf439","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.004210609Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.004225508Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382,"session_id":"a95fc09f-1534-4a3a-aa18-976c2c9bf439"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:54.004240827Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 487 len 16384 data: 5 4 4 4 02/30 Write block 127 len 4096 data: 4 03/30 Read block 502 len 28672 04/30 Read block 61 len 16384 05/30 Write block 194 len 12288 data: 5 4 4 06/30 Write block 582 len 36864 data: 5 6 4 4 2 2 2 2 2 07/30 Write block 445 len 28672 data: 5 6 5 4 5 4 4 08/30 Read block 487 len 12288 09/30 Flush 10/30 Write block 307 len 12288 data: 9 9 8 11/30 Read block 456 len 16384 12/30 Read block 63 len 28672 13/30 Write block 573 len 28672 data: 6 7 7 8 9 7 5 14/30 Read block 546 len 20480 15/30 Write block 298 len 12288 data: 6 5 5 16/30 Read block 176 len 16384 17/30 Read block 307 len 8192 18/30 Flush 19/30 Write block 461 len 40960 data: 3 2 3 3 4 5 4 3 3 3 20/30 Write block 132 len 24576 data: 3 3 4 4 4 4 21/30 Read block 45 len 8192 22/30 Read block 514 len 36864 23/30 Read block 443 len 16384 24/30 Flush 25/30 Write block 195 len 16384 data: 5 5 4 5 26/30 Read block 103 len 24576 27/30 Write block 523 len 32768 data: 6 6 6 6 5 4 4 4 28/30 Write block 463 len 28672 data: 4 4 5 6 5 4 4 29/30 Read block 22 len 36864 30/30 Write block 305 len 4096 data: 10 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 4 Done Done Done false 26 Acked 1025 Read 6 Done Done Done false 27 Acked 1026 Write 8 Done Done Done false 28 Acked 1027 Write 7 Sent Done Done false 29 Acked 1028 Read 9 Sent Done Sent false 30 Acked 1029 Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 1 2 6 Done 3 5 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:29:54.029349585Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1382} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 73 73 73 F F F 1 020-039 44 44 44 74 74 74 F F F 2 040-059 48 48 48 84 84 84 F F F 3 060-079 47 47 47 81 81 81 F F F 4 080-099 46 46 46 80 80 80 F F F 5 100-119 42 42 42 72 72 72 F F F 6 120-139 50 50 50 88 88 88 F F F 7 140-159 36 36 36 64 64 64 F F F 8 160-179 40 40 40 69 69 69 F F F 9 180-199 50 50 50 86 86 86 T T T 10 200-219 36 36 36 64 64 64 F F F 11 220-239 46 46 46 80 80 80 F F F 12 240-259 49 49 49 85 85 85 F F F 13 260-279 47 47 47 81 81 81 F F F 14 280-299 50 50 50 87 87 87 F F F 15 300-319 50 50 50 87 87 87 T T T 16 320-339 46 46 46 79 79 79 F F F 17 340-359 48 48 48 84 84 84 F F F 18 360-379 48 48 48 84 84 84 F F F 19 380-399 34 34 34 61 61 61 F F F 20 400-419 42 42 42 71 71 71 F F F 21 420-439 42 42 42 71 71 71 F F F 22 440-459 50 50 50 86 86 86 F F F 23 460-479 50 50 50 88 88 88 T T T 24 480-499 50 50 50 86 86 86 F F F 25 500-519 48 48 48 84 84 84 F F F 26 520-539 44 44 44 74 74 74 T T T 27 540-559 49 49 49 85 85 85 F F F 28 560-579 50 50 50 87 87 87 F F F 29 580-599 50 50 50 86 86 86 F F F Max gen: 50, Max flush: 88 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 73 73 73 F F F 1 020-039 44 44 44 74 74 74 F F F 2 040-059 48 48 48 84 84 84 F F F 3 060-079 47 47 47 81 81 81 F F F 4 080-099 46 46 46 80 80 80 F F F 5 100-119 42 42 42 72 72 72 F F F 6 120-139 50 50 50 88 88 88 F F F 7 140-159 36 36 36 64 64 64 F F F 8 160-179 40 40 40 69 69 69 F F F 9 180-199 51 51 51 89 89 89 F F F 10 200-219 36 36 36 64 64 64 F F F 11 220-239 46 46 46 80 80 80 F F F 12 240-259 49 49 49 85 85 85 F F F 13 260-279 47 47 47 81 81 81 F F F 14 280-299 50 50 50 87 87 87 F F F 15 300-319 51 51 51 89 89 89 F F F 16 320-339 46 46 46 79 79 79 F F F 17 340-359 48 48 48 84 84 84 F F F 18 360-379 48 48 48 84 84 84 F F F 19 380-399 34 34 34 61 61 61 F F F 20 400-419 42 42 42 71 71 71 F F F 21 420-439 42 42 42 71 71 71 F F F 22 440-459 50 50 50 86 86 86 F F F 23 460-479 51 51 51 89 89 89 F F F 24 480-499 50 50 50 86 86 86 F F F 25 500-519 48 48 48 84 84 84 F F F 26 520-539 51 51 51 89 89 89 F F F 27 540-559 49 49 49 85 85 85 F F F 28 560-579 50 50 50 87 87 87 F F F 29 580-599 50 50 50 86 86 86 F F F Max gen: 51, Max flush: 89 Begin loop 26 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.393674965Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.394026395Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.394060083Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.394095661Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390} {"msg":"Crucible 00db4809-c7f0-4c6c-8245-c75146a77182 has session id: bae8e2c8-06a5-401c-a006-5a2cbc40631a","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.394121799Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"Upstairs opts: Upstairs UUID: 00db4809-c7f0-4c6c-8245-c75146a77182, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.394168817Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"Crucible stats registered with UUID: 00db4809-c7f0-4c6c-8245-c75146a77182","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.394197835Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.425706601Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390} {"msg":"00db4809-c7f0-4c6c-8245-c75146a77182 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.425744079Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.425766827Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"client":"0","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.425811005Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"client":"0","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.425846283Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"client":"1","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.425880611Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"client":"1","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.425912829Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"client":"2","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.425946977Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"client":"2","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.425979205Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"":"io task","client":"2","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.426013033Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"":"io task","client":"0","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.426050581Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"ds_connection connected from Ok(127.0.0.1:48886)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.42607956Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"":"io task","client":"2","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"ds_connection connected from Ok(127.0.0.1:40517)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.426235941Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"":"io task","client":"0","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"ds_connection connected from Ok(127.0.0.1:61662)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.426264449Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.426590371Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"client":"2","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.426614839Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"client":"2","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.426630029Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"client":"0","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.426643718Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"client":"0","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.427056874Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"client":"1","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.427099452Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"client":"1","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.427267652Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"client":"2","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.42747895Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"client":"0","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.427794252Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"client":"1","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.427972062Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"client":"2","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.427991361Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"client":"2","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.42800926Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.428109165Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"client":"0","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.428127334Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"client":"0","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.428140683Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.428268896Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"client":"1","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.428287895Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"client":"1","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"[0]R flush_numbers[0..12]: [73, 74, 84, 81, 80, 72, 88, 64, 69, 89, 64, 80]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.428303264Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"":"downstairs","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"[0]R generation[0..12]: [43, 44, 48, 47, 46, 42, 50, 36, 40, 51, 36, 46]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.428317213Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"":"downstairs","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.428331302Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"":"downstairs","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"[1]R flush_numbers[0..12]: [73, 74, 84, 81, 80, 72, 88, 64, 69, 89, 64, 80]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.428345031Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"":"downstairs","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"[1]R generation[0..12]: [43, 44, 48, 47, 46, 42, 50, 36, 40, 51, 36, 46]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.428358411Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"":"downstairs","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.42837265Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"":"downstairs","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"[2]R flush_numbers[0..12]: [73, 74, 84, 81, 80, 72, 88, 64, 69, 89, 64, 80]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.428386349Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"":"downstairs","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"[2]R generation[0..12]: [43, 44, 48, 47, 46, 42, 50, 36, 40, 51, 36, 46]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.428399738Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"":"downstairs","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.428413597Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"":"downstairs","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.428430656Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"":"downstairs","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.428444126Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"":"downstairs","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"Next flush: 90","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.428456835Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"":"downstairs","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.428470534Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"":"downstairs","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-26T14:29:58.428483813Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.428497063Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"client":"0","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.428510652Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"client":"1","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.428523921Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"client":"2","":"downstairs","session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.42853735Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.42854968Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"00db4809-c7f0-4c6c-8245-c75146a77182 is now active with session: bae8e2c8-06a5-401c-a006-5a2cbc40631a","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.428562389Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.428574338Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390,"session_id":"bae8e2c8-06a5-401c-a006-5a2cbc40631a"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:29:58.428586458Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 169 len 36864 data: 2 4 3 3 3 3 3 3 2 02/30 Write block 286 len 28672 data: 7 5 6 7 7 6 7 03/30 Read block 514 len 16384 04/30 Read block 394 len 28672 05/30 Read block 563 len 8192 06/30 Flush 07/30 Flush 08/30 Write block 243 len 28672 data: 5 4 5 4 5 3 3 09/30 Read block 305 len 40960 10/30 Read block 124 len 16384 11/30 Read block 101 len 4096 12/30 Write block 465 len 28672 data: 6 7 6 5 5 4 4 13/30 Flush 14/30 Write block 491 len 36864 data: 4 4 4 5 5 5 5 6 6 15/30 Read block 292 len 4096 16/30 Write block 378 len 24576 data: 4 3 3 3 5 5 17/30 Read block 361 len 20480 18/30 Write block 298 len 8192 data: 7 6 19/30 Flush 20/30 Write block 187 len 28672 data: 5 5 4 4 5 6 5 21/30 Read block 413 len 8192 22/30 Write block 112 len 36864 data: 7 7 7 6 5 5 6 6 6 23/30 Write block 515 len 4096 data: 3 24/30 Write block 360 len 4096 data: 5 25/30 Read block 51 len 28672 26/30 Write block 277 len 12288 data: 8 9 8 27/30 Write block 229 len 8192 data: 4 5 28/30 Read block 573 len 20480 29/30 Read block 57 len 32768 30/30 Write block 397 len 24576 data: 7 8 7 6 7 6 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 20 Acked 1019 Write 7 Done Done Done false 21 Acked 1020 Read 2 Done Done Done false 22 Acked 1021 Write 9 Done Done Done false 23 Acked 1022 Write 1 Done Done Done false 24 Acked 1023 Write 1 Done Done Done false 25 Acked 1024 Read 7 Done Done Done false 26 Acked 1025 Write 3 Done Done Sent false 27 Acked 1026 Write 2 Done Done Sent false 28 Acked 1027 Read 5 Done Done Sent false 29 Acked 1028 Read 8 Done Done Sent false 30 Acked 1029 Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 5 7 Done 10 10 6 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:29:58.454426614Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1390} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 73 73 73 F F F 1 020-039 44 44 44 74 74 74 F F F 2 040-059 48 48 48 84 84 84 F F F 3 060-079 47 47 47 81 81 81 F F F 4 080-099 46 46 46 80 80 80 F F F 5 100-119 42 42 42 72 72 72 T T T 6 120-139 50 50 50 88 88 88 T T T 7 140-159 36 36 36 64 64 64 F F F 8 160-179 52 52 52 90 90 90 F F F 9 180-199 51 51 51 89 89 89 T T T 10 200-219 36 36 36 64 64 64 F F F 11 220-239 46 46 46 80 80 80 T T T 12 240-259 52 52 52 92 92 92 F F F 13 260-279 47 47 47 81 81 81 T T T 14 280-299 52 52 52 93 93 93 F F F 15 300-319 51 51 51 89 89 89 F F F 16 320-339 46 46 46 79 79 79 F F F 17 340-359 48 48 48 84 84 84 F F F 18 360-379 52 52 52 93 93 93 T T T 19 380-399 52 52 52 93 93 93 T T T 20 400-419 42 42 42 71 71 71 T T T 21 420-439 42 42 42 71 71 71 F F F 22 440-459 50 50 50 86 86 86 F F F 23 460-479 52 52 52 92 92 92 F F F 24 480-499 52 52 52 93 93 93 F F F 25 500-519 48 48 48 84 84 84 T T T 26 520-539 51 51 51 89 89 89 F F F 27 540-559 49 49 49 85 85 85 F F F 28 560-579 50 50 50 87 87 87 F F F 29 580-599 50 50 50 86 86 86 F F F Max gen: 52, Max flush: 93 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 73 73 73 F F F 1 020-039 44 44 44 74 74 74 F F F 2 040-059 48 48 48 84 84 84 F F F 3 060-079 47 47 47 81 81 81 F F F 4 080-099 46 46 46 80 80 80 F F F 5 100-119 53 53 53 94 94 94 F F F 6 120-139 53 53 53 94 94 94 F F F 7 140-159 36 36 36 64 64 64 F F F 8 160-179 52 52 52 90 90 90 F F F 9 180-199 53 53 53 94 94 94 F F F 10 200-219 36 36 36 64 64 64 F F F 11 220-239 53 53 53 94 94 94 F F F 12 240-259 52 52 52 92 92 92 F F F 13 260-279 53 53 53 94 94 94 F F F 14 280-299 52 52 52 93 93 93 F F F 15 300-319 51 51 51 89 89 89 F F F 16 320-339 46 46 46 79 79 79 F F F 17 340-359 48 48 48 84 84 84 F F F 18 360-379 53 53 53 94 94 94 F F F 19 380-399 53 53 53 94 94 94 F F F 20 400-419 53 53 53 94 94 94 F F F 21 420-439 42 42 42 71 71 71 F F F 22 440-459 50 50 50 86 86 86 F F F 23 460-479 52 52 52 92 92 92 F F F 24 480-499 52 52 52 93 93 93 F F F 25 500-519 53 53 53 94 94 94 F F F 26 520-539 51 51 51 89 89 89 F F F 27 540-559 49 49 49 85 85 85 F F F 28 560-579 50 50 50 87 87 87 F F F 29 580-599 50 50 50 86 86 86 F F F Max gen: 53, Max flush: 94 Begin loop 27 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.854204475Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.85446648Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.854487809Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.854503498Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398} {"msg":"Crucible ecb1d576-5330-43b6-ab5d-231152c3dc7c has session id: eda52883-46da-4ebc-b537-19dac63879c8","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.854671429Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"Upstairs opts: Upstairs UUID: ecb1d576-5330-43b6-ab5d-231152c3dc7c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.854719646Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"Crucible stats registered with UUID: ecb1d576-5330-43b6-ab5d-231152c3dc7c","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.854743134Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.885762178Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398} {"msg":"ecb1d576-5330-43b6-ab5d-231152c3dc7c active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.885793286Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.885811385Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"client":"0","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.885838893Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"client":"0","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.885856482Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"client":"1","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.885873601Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"client":"1","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.88589023Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"client":"2","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.88590686Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"client":"2","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.885923039Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.885945967Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"":"io task","client":"0","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.885978756Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"":"io task","client":"1","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"ds_connection connected from Ok(127.0.0.1:45534)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.886100018Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"":"io task","client":"0","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"ds_connection connected from Ok(127.0.0.1:35193)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.886297787Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"":"io task","client":"1","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"ds_connection connected from Ok(127.0.0.1:46547)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.886336705Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.886869015Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"client":"0","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.886900243Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"client":"0","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.88695989Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"client":"1","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.886979619Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"client":"1","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.886993078Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"client":"2","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.887036005Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"client":"2","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.887541227Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"client":"0","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.887572515Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"client":"1","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.887701218Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"client":"2","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.888227448Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"client":"0","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.888263016Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"client":"0","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.888279565Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.888292654Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"client":"1","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.888306404Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"client":"1","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.888319943Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.888332032Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"client":"2","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.888345241Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"client":"2","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"[0]R flush_numbers[0..12]: [73, 74, 84, 81, 80, 94, 94, 64, 90, 94, 64, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.88836291Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"":"downstairs","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"[0]R generation[0..12]: [43, 44, 48, 47, 46, 53, 53, 36, 52, 53, 36, 53]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.888377309Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"":"downstairs","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.888391559Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"":"downstairs","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"[1]R flush_numbers[0..12]: [73, 74, 84, 81, 80, 94, 94, 64, 90, 94, 64, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.888405448Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"":"downstairs","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"[1]R generation[0..12]: [43, 44, 48, 47, 46, 53, 53, 36, 52, 53, 36, 53]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.888419217Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"":"downstairs","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.888433237Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"":"downstairs","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"[2]R flush_numbers[0..12]: [73, 74, 84, 81, 80, 94, 94, 64, 90, 94, 64, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.888446706Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"":"downstairs","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"[2]R generation[0..12]: [43, 44, 48, 47, 46, 53, 53, 36, 52, 53, 36, 53]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.888460155Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"":"downstairs","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.888474034Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"":"downstairs","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"Wait for a query_work_queue command to finish before sending IO } {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.888490593Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"":"downstairs","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.888503942Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"":"downstairs","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"Next flush: 95","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.888516982Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"":"downstairs","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.888530111Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"":"downstairs","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.88854315Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.888557039Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"client":"0","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.888570529Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"client":"1","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.888593947Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"client":"2","":"downstairs","session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.888616736Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.888630845Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"ecb1d576-5330-43b6-ab5d-231152c3dc7c is now active with session: eda52883-46da-4ebc-b537-19dac63879c8","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.888644005Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.888655974Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398,"session_id":"eda52883-46da-4ebc-b537-19dac63879c8"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:02.888669993Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 390 len 20480 data: 2 2 2 2 3 02/30 Write block 136 len 8192 data: 5 5 03/30 Write block 460 len 8192 data: 4 4 04/30 Read block 184 len 32768 05/30 Read block 109 len 28672 06/30 Read block 21 len 32768 07/30 Read block 472 len 36864 08/30 Write block 99 len 12288 data: 2 3 4 09/30 Write block 182 len 12288 data: 7 8 7 10/30 Read block 570 len 20480 11/30 Read block 556 len 12288 12/30 Read block 141 len 20480 13/30 Write block 517 len 16384 data: 4 5 5 5 14/30 Write block 243 len 28672 data: 6 5 6 5 6 4 4 15/30 Write block 380 len 36864 data: 4 4 6 6 4 4 3 3 2 16/30 Flush 17/30 Write block 364 len 8192 data: 3 2 18/30 Write block 57 len 8192 data: 10 8 19/30 Read block 511 len 24576 20/30 Write block 13 len 16384 data: 6 6 6 6 21/30 Read block 270 len 24576 22/30 Read block 181 len 32768 23/30 Read block 80 len 8192 24/30 Read block 362 len 12288 25/30 Flush 26/30 Write block 184 len 8192 data: 8 7 27/30 Read block 525 len 8192 28/30 Read block 56 len 8192 29/30 Write block 449 len 40960 data: 6 5 5 4 4 6 7 7 7 6 30/30 Write block 319 len 20480 data: 5 4 3 3 3 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 2 Done Done Done false 27 Acked 1026 Read 2 Done Done Done false 28 Acked 1027 Read 2 Done Done Done false 29 Acked 1028 Write 10 Sent Sent Sent false 30 Acked 1029 Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:30:02.911246684Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1398} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 96 96 96 F F F 1 020-039 44 44 44 74 74 74 F F F 2 040-059 54 54 54 96 96 96 F F F 3 060-079 47 47 47 81 81 81 F F F 4 080-099 54 54 54 95 95 95 F F F 5 100-119 54 54 54 95 95 95 F F F 6 120-139 54 54 54 95 95 95 F F F 7 140-159 36 36 36 64 64 64 F F F 8 160-179 52 52 52 90 90 90 F F F 9 180-199 54 54 54 95 95 95 T T T 10 200-219 36 36 36 64 64 64 F F F 11 220-239 53 53 53 94 94 94 F F F 12 240-259 54 54 54 95 95 95 F F F 13 260-279 53 53 53 94 94 94 F F F 14 280-299 52 52 52 93 93 93 F F F 15 300-319 51 51 51 89 89 89 T T T 16 320-339 46 46 46 79 79 79 T T T 17 340-359 48 48 48 84 84 84 F F F 18 360-379 54 54 54 96 96 96 F F F 19 380-399 54 54 54 95 95 95 F F F 20 400-419 53 53 53 94 94 94 F F F 21 420-439 42 42 42 71 71 71 F F F 22 440-459 50 50 50 86 86 86 T T T 23 460-479 54 54 54 95 95 95 F F F 24 480-499 52 52 52 93 93 93 F F F 25 500-519 54 54 54 95 95 95 F F F 26 520-539 54 54 54 95 95 95 F F F 27 540-559 49 49 49 85 85 85 F F F 28 560-579 50 50 50 87 87 87 F F F 29 580-599 50 50 50 86 86 86 F F F Max gen: 54, Max flush: 96 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 96 96 96 F F F 1 020-039 44 44 44 74 74 74 F F F 2 040-059 54 54 54 96 96 96 F F F 3 060-079 47 47 47 81 81 81 F F F 4 080-099 54 54 54 95 95 95 F F F 5 100-119 54 54 54 95 95 95 F F F 6 120-139 54 54 54 95 95 95 F F F 7 140-159 36 36 36 64 64 64 F F F 8 160-179 52 52 52 90 90 90 F F F 9 180-199 55 55 55 97 97 97 F F F 10 200-219 36 36 36 64 64 64 F F F 11 220-239 53 53 53 94 94 94 F F F 12 240-259 54 54 54 95 95 95 F F F 13 260-279 53 53 53 94 94 94 F F F 14 280-299 52 52 52 93 93 93 F F F 15 300-319 55 55 55 97 97 97 F F F 16 320-339 55 55 55 97 97 97 F F F 17 340-359 48 48 48 84 84 84 F F F 18 360-379 54 54 54 96 96 96 F F F 19 380-399 54 54 54 95 95 95 F F F 20 400-419 53 53 53 94 94 94 F F F 21 420-439 42 42 42 71 71 71 F F F 22 440-459 55 55 55 97 97 97 F F F 23 460-479 54 54 54 95 95 95 F F F 24 480-499 52 52 52 93 93 93 F F F 25 500-519 54 54 54 95 95 95 F F F 26 520-539 54 54 54 95 95 95 F F F 27 540-559 49 49 49 85 85 85 F F F 28 560-579 50 50 50 87 87 87 F F F 29 580-599 50 50 50 86 86 86 F F F Max gen: 55, Max flush: 97 Begin loop 28 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.250163692Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.250446006Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.250467165Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.250496863Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406} {"msg":"Crucible ebeffef6-4774-4e88-a852-53997b846ac6 has session id: 0f237b06-a4d8-4c1e-8798-79918fcc96ff","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.250656365Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"Upstairs opts: Upstairs UUID: ebeffef6-4774-4e88-a852-53997b846ac6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.250714581Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"Crucible stats registered with UUID: ebeffef6-4774-4e88-a852-53997b846ac6","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.25074332Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.282179409Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406} {"msg":"ebeffef6-4774-4e88-a852-53997b846ac6 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.282212967Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.282231676Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"client":"0","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.282258885Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"client":"0","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.282276534Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"client":"1","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.282293523Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"client":"1","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.282310242Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"client":"2","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.28233507Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"client":"2","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.282364379Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.282394267Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.282423535Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:51393)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.282455204Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:38368)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.282486752Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:64102)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.2825186Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.283250999Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"client":"2","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.283286306Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"client":"2","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.283314445Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"client":"0","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.283340953Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"client":"0","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.28340941Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"client":"1","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.283444568Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"client":"1","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.283945559Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"client":"0","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.283972748Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"client":"1","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.284025025Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"client":"2","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.284571244Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"client":"2","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.284598302Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"client":"2","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.284615121Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.28462803Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"client":"1","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.28464143Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"client":"1","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.284654879Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.284667238Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"client":"0","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.284680678Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"client":"0","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"[0]R flush_numbers[0..12]: [96, 74, 96, 81, 95, 95, 95, 64, 90, 97, 64, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.284694527Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"":"downstairs","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"[0]R generation[0..12]: [54, 44, 54, 47, 54, 54, 54, 36, 52, 55, 36, 53]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.284708436Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"":"downstairs","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.284722435Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"":"downstairs","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"[1]R flush_numbers[0..12]: [96, 74, 96, 81, 95, 95, 95, 64, 90, 97, 64, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.284736104Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"":"downstairs","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"[1]R generation[0..12]: [54, 44, 54, 47, 54, 54, 54, 36, 52, 55, 36, 53]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.284749634Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"":"downstairs","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.284763573Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"":"downstairs","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"[2]R flush_numbers[0..12]: [96, 74, 96, 81, 95, 95, 95, 64, 90, 97, 64, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.284781722Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"":"downstairs","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"[2]R generation[0..12]: [54, 44, 54, 47, 54, 54, 54, 36, 52, 55, 36, 53]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.284795701Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"":"downstairs","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.28480972Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"":"downstairs","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.284822499Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"":"downstairs","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.284835649Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"":"downstairs","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"Next flush: 98","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.284848298Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"":"downstairs","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.284861187Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"":"downstairs","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.284874867Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.284886886Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"client":"0","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.284904365Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"client":"1","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.284918074Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"client":"2","":"downstairs","session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.284931123Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.284943323Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"ebeffef6-4774-4e88-a852-53997b846ac6 is now active with session: 0f237b06-a4d8-4c1e-8798-79918fcc96ff","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.284956192Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.284968101Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406,"session_id":"0f237b06-a4d8-4c1e-8798-79918fcc96ff"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:07.284982321Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 108 len 24576 data: 5 4 4 6 8 8 02/30 Read block 589 len 40960 03/30 Read block 422 len 32768 04/30 Read block 101 len 8192 05/30 Write block 479 len 28672 data: 7 7 7 6 6 5 6 06/30 Write block 493 len 36864 data: 5 6 6 6 6 7 7 7 7 07/30 Write block 313 len 8192 data: 8 7 08/30 Read block 28 len 12288 09/30 Flush 10/30 Write block 86 len 32768 data: 5 5 5 5 5 5 4 4 11/30 Read block 189 len 28672 12/30 Read block 358 len 16384 13/30 Write block 537 len 36864 data: 5 6 5 5 4 3 4 5 6 14/30 Write block 66 len 4096 data: 8 15/30 Read block 62 len 40960 16/30 Read block 463 len 36864 17/30 Flush 18/30 Write block 282 len 4096 data: 7 19/30 Read block 41 len 12288 20/30 Read block 514 len 40960 21/30 Write block 72 len 40960 data: 6 6 4 3 5 4 5 6 5 6 22/30 Read block 196 len 4096 23/30 Read block 161 len 28672 24/30 Write block 72 len 12288 data: 7 7 5 25/30 Write block 430 len 36864 data: 5 5 5 5 4 4 3 3 3 26/30 Read block 385 len 16384 27/30 Read block 130 len 28672 28/30 Write block 556 len 32768 data: 5 4 4 3 4 3 3 3 29/30 Write block 317 len 32768 data: 7 5 6 5 4 4 4 5 30/30 Write block 156 len 8192 data: 3 3 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 18 Acked 1017 Write 1 Done Done Done false 19 Acked 1018 Read 3 Done Done Done false 20 Acked 1019 Read 10 Done Done Done false 21 Acked 1020 Write 10 Done Done Done false 22 Acked 1021 Read 1 Done Done Done false 23 Acked 1022 Read 7 Done Done Done false 24 Acked 1023 Write 3 Done Done Done false 25 Acked 1024 Write 9 Done Done Done false 26 Acked 1025 Read 4 Done Done Done false 27 Acked 1026 Read 7 Done Done Done false 28 Acked 1027 Write 8 Sent Sent Sent false 29 Acked 1028 Write 8 Sent Sent Sent false 30 Acked 1029 Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:30:07.308853818Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1406} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 96 96 96 F F F 1 020-039 44 44 44 74 74 74 F F F 2 040-059 54 54 54 96 96 96 F F F 3 060-079 56 56 56 99 99 99 T T T 4 080-099 56 56 56 99 99 99 T T T 5 100-119 56 56 56 98 98 98 F F F 6 120-139 54 54 54 95 95 95 F F F 7 140-159 36 36 36 64 64 64 T T T 8 160-179 52 52 52 90 90 90 F F F 9 180-199 55 55 55 97 97 97 F F F 10 200-219 36 36 36 64 64 64 F F F 11 220-239 53 53 53 94 94 94 F F F 12 240-259 54 54 54 95 95 95 F F F 13 260-279 53 53 53 94 94 94 F F F 14 280-299 52 52 52 93 93 93 T T T 15 300-319 56 56 56 98 98 98 T T T 16 320-339 55 55 55 97 97 97 T T T 17 340-359 48 48 48 84 84 84 F F F 18 360-379 54 54 54 96 96 96 F F F 19 380-399 54 54 54 95 95 95 F F F 20 400-419 53 53 53 94 94 94 F F F 21 420-439 42 42 42 71 71 71 T T T 22 440-459 55 55 55 97 97 97 F F F 23 460-479 56 56 56 98 98 98 F F F 24 480-499 56 56 56 98 98 98 F F F 25 500-519 56 56 56 98 98 98 F F F 26 520-539 56 56 56 99 99 99 F F F 27 540-559 56 56 56 99 99 99 T T T 28 560-579 50 50 50 87 87 87 T T T 29 580-599 50 50 50 86 86 86 F F F Max gen: 56, Max flush: 99 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 96 96 96 F F F 1 020-039 44 44 44 74 74 74 F F F 2 040-059 54 54 54 96 96 96 F F F 3 060-079 57 57 57 100 100 100 F F F 4 080-099 57 57 57 100 100 100 F F F 5 100-119 56 56 56 98 98 98 F F F 6 120-139 54 54 54 95 95 95 F F F 7 140-159 57 57 57 100 100 100 F F F 8 160-179 52 52 52 90 90 90 F F F 9 180-199 55 55 55 97 97 97 F F F 10 200-219 36 36 36 64 64 64 F F F 11 220-239 53 53 53 94 94 94 F F F 12 240-259 54 54 54 95 95 95 F F F 13 260-279 53 53 53 94 94 94 F F F 14 280-299 57 57 57 100 100 100 F F F 15 300-319 57 57 57 100 100 100 F F F 16 320-339 57 57 57 100 100 100 F F F 17 340-359 48 48 48 84 84 84 F F F 18 360-379 54 54 54 96 96 96 F F F 19 380-399 54 54 54 95 95 95 F F F 20 400-419 53 53 53 94 94 94 F F F 21 420-439 57 57 57 100 100 100 F F F 22 440-459 55 55 55 97 97 97 F F F 23 460-479 56 56 56 98 98 98 F F F 24 480-499 56 56 56 98 98 98 F F F 25 500-519 56 56 56 98 98 98 F F F 26 520-539 56 56 56 99 99 99 F F F 27 540-559 57 57 57 100 100 100 F F F 28 560-579 57 57 57 100 100 100 F F F 29 580-599 50 50 50 86 86 86 F F F Max gen: 57, Max flush: 100 Begin loop 29 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.71180171Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.712138321Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.712167199Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.712196608Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414} {"msg":"Crucible 513d2f66-8650-4e76-893d-82431630dcb8 has session id: 52cbfe47-03e1-4f3f-ba57-be3ed2c96252","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.712233865Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"Upstairs opts: Upstairs UUID: 513d2f66-8650-4e76-893d-82431630dcb8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.712274203Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"Crucible stats registered with UUID: 513d2f66-8650-4e76-893d-82431630dcb8","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.712297392Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.743444068Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414} {"msg":"513d2f66-8650-4e76-893d-82431630dcb8 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.743477936Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.743504314Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"client":"0","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.743549722Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"client":"0","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.7435872Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"client":"1","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.743621898Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"client":"1","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.743657726Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"client":"2","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.743691684Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"client":"2","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.743725222Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.74375932Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.743796738Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"":"io task","client":"1","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"ds_connection connected from Ok(127.0.0.1:39537)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.743889653Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"ds_connection connected from Ok(127.0.0.1:44515)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.743921741Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"":"io task","client":"1","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"ds_connection connected from Ok(127.0.0.1:50101)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.74394199Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.744504808Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"client":"0","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.744540236Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"client":"0","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.744568004Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"client":"2","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.744593573Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"client":"2","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.744622951Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"client":"1","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.74465157Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"client":"1","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.74516601Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"client":"0","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.745302182Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"client":"1","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.745331181Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"client":"2","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.745742668Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"client":"0","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.745765206Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"client":"0","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.745779706Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.745884489Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"client":"2","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.745904728Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"client":"2","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.745918418Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.745976454Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"client":"1","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.745995613Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"client":"1","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"[0]R flush_numbers[0..12]: [96, 74, 96, 100, 100, 98, 95, 100, 90, 97, 64, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.74604851Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"":"downstairs","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"[0]R generation[0..12]: [54, 44, 54, 57, 57, 56, 54, 57, 52, 55, 36, 53]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.746074089Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"":"downstairs","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.746097868Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"":"downstairs","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"[1]R flush_numbers[0..12]: [96, 74, 96, 100, 100, 98, 95, 100, 90, 97, 64, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.746112727Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"":"downstairs","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"[1]R generation[0..12]: [54, 44, 54, 57, 57, 56, 54, 57, 52, 55, 36, 53]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.746126416Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"":"downstairs","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.746140585Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"":"downstairs","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"[2]R flush_numbers[0..12]: [96, 74, 96, 100, 100, 98, 95, 100, 90, 97, 64, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.746154344Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"":"downstairs","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"[2]R generation[0..12]: [54, 44, 54, 57, 57, 56, 54, 57, 52, 55, 36, 53]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.746167864Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"":"downstairs","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.746181843Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"":"downstairs","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.746197742Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"":"downstairs","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.746211251Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"":"downstairs","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"Next flush: 101","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-26T14:30:11.74622444Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"":"downstairs","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.74624088Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"":"downstairs","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.746253949Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.746265938Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"client":"0","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.746279037Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"client":"1","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.746292137Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"client":"2","":"downstairs","session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.746305306Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.746317495Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"513d2f66-8650-4e76-893d-82431630dcb8 is now active with session: 52cbfe47-03e1-4f3f-ba57-be3ed2c96252","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.746330324Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-09-26T14:30:11.746342434Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414,"session_id":"52cbfe47-03e1-4f3f-ba57-be3ed2c96252"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:11.746356813Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 196 len 24576 data: 6 5 6 4 5 4 02/30 Read block 212 len 40960 03/30 Write block 90 len 16384 data: 6 6 5 5 04/30 Read block 422 len 12288 05/30 Read block 341 len 20480 06/30 Read block 408 len 12288 07/30 Read block 321 len 32768 08/30 Read block 322 len 4096 09/30 Read block 287 len 4096 10/30 Read block 91 len 32768 11/30 Read block 569 len 28672 12/30 Read block 22 len 16384 13/30 Write block 73 len 40960 data: 8 6 4 6 5 6 7 6 7 6 14/30 Read block 17 len 4096 15/30 Write block 156 len 16384 data: 4 4 4 5 16/30 Write block 359 len 36864 data: 5 6 4 4 3 4 3 2 3 17/30 Write block 155 len 36864 data: 3 5 5 5 6 3 3 3 3 18/30 Write block 0 len 16384 data: 4 4 3 4 19/30 Flush 20/30 Write block 499 len 20480 data: 8 8 8 9 6 21/30 Read block 526 len 8192 22/30 Read block 326 len 36864 23/30 Write block 213 len 32768 data: 5 5 4 5 6 5 5 4 24/30 Write block 478 len 16384 data: 4 8 8 8 25/30 Read block 502 len 40960 26/30 Write block 325 len 20480 data: 5 4 5 6 5 27/30 Write block 569 len 40960 data: 5 5 5 5 7 8 8 9 10 8 28/30 Write block 107 len 12288 data: 4 6 5 29/30 Read block 597 len 12288 30/30 Read block 440 len 28672 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 20 Acked 1019 Write 5 Done Done Done false 21 Acked 1020 Read 2 Done Done Done false 22 Acked 1021 Read 9 Done Done Done false 23 Acked 1022 Write 8 Done Done Done false 24 Acked 1023 Write 4 Done Done Done false 25 Acked 1024 Read 10 Done Done Done false 26 Acked 1025 Write 5 Done Done Done false 27 Acked 1026 Write 10 Done Done Done false 28 Acked 1027 Write 3 Sent Done Sent false 29 Acked 1028 Read 3 Sent Done Sent false 30 Acked 1029 Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 0 3 6 Done 8 11 8 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:30:11.76882675Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1414} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 101 101 101 F F F 1 020-039 44 44 44 74 74 74 F F F 2 040-059 54 54 54 96 96 96 F F F 3 060-079 58 58 58 101 101 101 F F F 4 080-099 58 58 58 101 101 101 F F F 5 100-119 56 56 56 98 98 98 T T T 6 120-139 54 54 54 95 95 95 F F F 7 140-159 58 58 58 101 101 101 F F F 8 160-179 58 58 58 101 101 101 F F F 9 180-199 58 58 58 101 101 101 F F F 10 200-219 58 58 58 101 101 101 T T T 11 220-239 53 53 53 94 94 94 T T T 12 240-259 54 54 54 95 95 95 F F F 13 260-279 53 53 53 94 94 94 F F F 14 280-299 57 57 57 100 100 100 F F F 15 300-319 57 57 57 100 100 100 F F F 16 320-339 57 57 57 100 100 100 T T T 17 340-359 58 58 58 101 101 101 F F F 18 360-379 58 58 58 101 101 101 F F F 19 380-399 54 54 54 95 95 95 F F F 20 400-419 53 53 53 94 94 94 F F F 21 420-439 57 57 57 100 100 100 F F F 22 440-459 55 55 55 97 97 97 F F F 23 460-479 56 56 56 98 98 98 T T T 24 480-499 56 56 56 98 98 98 T T T 25 500-519 56 56 56 98 98 98 T T T 26 520-539 56 56 56 99 99 99 F F F 27 540-559 57 57 57 100 100 100 F F F 28 560-579 57 57 57 100 100 100 T T T 29 580-599 50 50 50 86 86 86 F F F Max gen: 58, Max flush: 101 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 101 101 101 F F F 1 020-039 44 44 44 74 74 74 F F F 2 040-059 54 54 54 96 96 96 F F F 3 060-079 58 58 58 101 101 101 F F F 4 080-099 58 58 58 101 101 101 F F F 5 100-119 59 59 59 102 102 102 F F F 6 120-139 54 54 54 95 95 95 F F F 7 140-159 58 58 58 101 101 101 F F F 8 160-179 58 58 58 101 101 101 F F F 9 180-199 58 58 58 101 101 101 F F F 10 200-219 59 59 59 102 102 102 F F F 11 220-239 59 59 59 102 102 102 F F F 12 240-259 54 54 54 95 95 95 F F F 13 260-279 53 53 53 94 94 94 F F F 14 280-299 57 57 57 100 100 100 F F F 15 300-319 57 57 57 100 100 100 F F F 16 320-339 59 59 59 102 102 102 F F F 17 340-359 58 58 58 101 101 101 F F F 18 360-379 58 58 58 101 101 101 F F F 19 380-399 54 54 54 95 95 95 F F F 20 400-419 53 53 53 94 94 94 F F F 21 420-439 57 57 57 100 100 100 F F F 22 440-459 55 55 55 97 97 97 F F F 23 460-479 59 59 59 102 102 102 F F F 24 480-499 59 59 59 102 102 102 F F F 25 500-519 59 59 59 102 102 102 F F F 26 520-539 56 56 56 99 99 99 F F F 27 540-559 57 57 57 100 100 100 F F F 28 560-579 59 59 59 102 102 102 F F F 29 580-599 50 50 50 86 86 86 F F F Max gen: 59, Max flush: 102 Begin loop 30 Downstairs to restart: 2 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.163971733Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.164274016Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.164297224Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.164323863Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422} {"msg":"Crucible 4dd296ea-9f13-491a-a525-5cd94ed3d3ac has session id: 0797c29c-9ba7-4e9f-ad40-239a22a415e9","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.164507993Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"Upstairs opts: Upstairs UUID: 4dd296ea-9f13-491a-a525-5cd94ed3d3ac, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.164567449Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"Crucible stats registered with UUID: 4dd296ea-9f13-491a-a525-5cd94ed3d3ac","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.164588408Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.197193521Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422} {"msg":"4dd296ea-9f13-491a-a525-5cd94ed3d3ac active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.197226649Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.197243879Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"client":"0","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.197271067Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"client":"0","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.197288366Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"client":"1","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.197305015Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"client":"1","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.197361642Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"client":"2","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.19739032Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"client":"2","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.197410549Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.197425518Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"":"io task","client":"0","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.197440627Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"":"io task","client":"1","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:50021)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.197533762Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"":"io task","client":"0","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:40940)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.19756873Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"":"io task","client":"1","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:45467)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.197776718Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.19828018Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"client":"0","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.198354976Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"client":"0","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.198392303Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"client":"1","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.198423682Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"client":"1","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.19844658Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"client":"2","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.19846107Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"client":"2","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.198956931Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"client":"0","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.199114453Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"client":"1","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.19915478Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"client":"2","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.199495381Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"client":"0","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.1995199Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"client":"0","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.199535979Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.199709489Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"client":"1","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.199734827Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"client":"1","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.199749977Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.199762406Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"client":"2","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.199778915Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"client":"2","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"[0]R flush_numbers[0..12]: [101, 74, 96, 101, 101, 102, 95, 101, 101, 101, 102, 102]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.199793394Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"":"downstairs","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"[0]R generation[0..12]: [58, 44, 54, 58, 58, 59, 54, 58, 58, 58, 59, 59]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.199807263Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"":"downstairs","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.199821382Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"":"downstairs","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"[1]R flush_numbers[0..12]: [101, 74, 96, 101, 101, 102, 95, 101, 101, 101, 102, 102]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.199834912Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"":"downstairs","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"[1]R generation[0..12]: [58, 44, 54, 58, 58, 59, 54, 58, 58, 58, 59, 59]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.199848331Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"":"downstairs","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.19986235Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"":"downstairs","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"[2]R flush_numbers[0..12]: [101, 74, 96, 101, 101, 102, 95, 101, 101, 101, 102, 102]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.199876489Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"":"downstairs","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"[2]R generation[0..12]: [58, 44, 54, 58, 58, 59, 54, 58, 58, 58, 59, 59]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.199890009Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"":"downstairs","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.199903728Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"":"downstairs","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.199916647Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"":"downstairs","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.199929906Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"":"downstairs","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"Next flush: 103","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.199942746Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"":"downstairs","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.199955565Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"":"downstairs","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"Wait for a query_work_queue command to finish before sending IO No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.199974014Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.199986123Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"client":"0","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.200007712Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"client":"1","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.200031291Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"client":"2","":"downstairs","session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.20004585Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.200058079Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"4dd296ea-9f13-491a-a525-5cd94ed3d3ac is now active with session: 0797c29c-9ba7-4e9f-ad40-239a22a415e9","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.200070878Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.200082678Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422,"session_id":"0797c29c-9ba7-4e9f-ad40-239a22a415e9"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:16.200101907Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 588 len 8192 data: 3 3 02/30 Write block 165 len 36864 data: 4 3 4 4 3 5 4 4 4 03/30 Read block 66 len 16384 04/30 Write block 179 len 40960 data: 5 7 6 8 9 9 8 5 6 6 05/30 Read block 356 len 36864 06/30 Read block 191 len 16384 07/30 Flush 08/30 Write block 298 len 12288 data: 8 7 6 09/30 Write block 101 len 4096 data: 5 10/30 Read block 396 len 20480 11/30 Write block 347 len 32768 data: 7 7 6 5 7 7 8 6 12/30 Read block 154 len 8192 13/30 Read block 255 len 20480 14/30 Write block 574 len 8192 data: 9 9 15/30 Read block 334 len 12288 16/30 Flush 17/30 Write block 540 len 32768 data: 6 5 4 5 6 7 6 5 18/30 Write block 558 len 4096 data: 5 19/30 Write block 129 len 20480 data: 4 4 3 4 4 20/30 Read block 585 len 4096 21/30 Read block 3 len 36864 22/30 Read block 230 len 8192 23/30 Flush 24/30 Write block 503 len 12288 data: 7 6 6 25/30 Write block 147 len 4096 data: 3 26/30 Write block 111 len 16384 data: 7 9 9 8 27/30 Read block 465 len 40960 28/30 Write block 378 len 12288 data: 5 4 5 29/30 Write block 287 len 32768 data: 6 7 8 8 7 8 7 8 30/30 Read block 360 len 12288 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 3 Done Done Done false 25 Acked 1024 Write 1 Done Done Done false 26 Acked 1025 Write 4 Done Done Done false 27 Acked 1026 Read 10 Done Done Done false 28 Acked 1027 Write 3 Done Done Done false 29 Acked 1028 Write 8 Done Sent Done false 30 Acked 1029 Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 1 3 Done 7 5 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:30:16.223898399Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1422} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 101 101 101 F F F 1 020-039 44 44 44 74 74 74 F F F 2 040-059 54 54 54 96 96 96 F F F 3 060-079 58 58 58 101 101 101 F F F 4 080-099 58 58 58 101 101 101 F F F 5 100-119 60 60 60 104 104 104 T T T 6 120-139 60 60 60 105 105 105 F F F 7 140-159 58 58 58 101 101 101 T T T 8 160-179 60 60 60 103 103 103 F F F 9 180-199 60 60 60 103 103 103 F F F 10 200-219 59 59 59 102 102 102 F F F 11 220-239 59 59 59 102 102 102 F F F 12 240-259 54 54 54 95 95 95 F F F 13 260-279 53 53 53 94 94 94 F F F 14 280-299 60 60 60 104 104 104 T T T 15 300-319 60 60 60 104 104 104 F F F 16 320-339 59 59 59 102 102 102 F F F 17 340-359 60 60 60 104 104 104 F F F 18 360-379 58 58 58 101 101 101 T T T 19 380-399 54 54 54 95 95 95 T T T 20 400-419 53 53 53 94 94 94 F F F 21 420-439 57 57 57 100 100 100 F F F 22 440-459 55 55 55 97 97 97 F F F 23 460-479 59 59 59 102 102 102 F F F 24 480-499 59 59 59 102 102 102 F F F 25 500-519 59 59 59 102 102 102 T T T 26 520-539 56 56 56 99 99 99 F F F 27 540-559 60 60 60 105 105 105 F F F 28 560-579 60 60 60 104 104 104 F F F 29 580-599 60 60 60 103 103 103 F F F Max gen: 60, Max flush: 105 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 101 101 101 F F F 1 020-039 44 44 44 74 74 74 F F F 2 040-059 54 54 54 96 96 96 F F F 3 060-079 58 58 58 101 101 101 F F F 4 080-099 58 58 58 101 101 101 F F F 5 100-119 61 61 61 106 106 106 F F F 6 120-139 60 60 60 105 105 105 F F F 7 140-159 61 61 61 106 106 106 F F F 8 160-179 60 60 60 103 103 103 F F F 9 180-199 60 60 60 103 103 103 F F F 10 200-219 59 59 59 102 102 102 F F F 11 220-239 59 59 59 102 102 102 F F F 12 240-259 54 54 54 95 95 95 F F F 13 260-279 53 53 53 94 94 94 F F F 14 280-299 61 61 61 106 106 106 F F F 15 300-319 60 60 60 104 104 104 F F F 16 320-339 59 59 59 102 102 102 F F F 17 340-359 60 60 60 104 104 104 F F F 18 360-379 61 61 61 106 106 106 F F F 19 380-399 61 61 61 106 106 106 F F F 20 400-419 53 53 53 94 94 94 F F F 21 420-439 57 57 57 100 100 100 F F F 22 440-459 55 55 55 97 97 97 F F F 23 460-479 59 59 59 102 102 102 F F F 24 480-499 59 59 59 102 102 102 F F F 25 500-519 61 61 61 106 106 106 F F F 26 520-539 56 56 56 99 99 99 F F F 27 540-559 60 60 60 105 105 105 F F F 28 560-579 60 60 60 104 104 104 F F F 29 580-599 60 60 60 103 103 103 F F F Max gen: 61, Max flush: 106 Begin loop 31 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.587514879Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.587716247Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.587732356Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.587747486Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430} {"msg":"Crucible f5827950-7386-4698-9959-8707626ea7b5 has session id: 8ed46881-1de8-4681-9977-75570486b6d4","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.587948274Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"Upstairs opts: Upstairs UUID: f5827950-7386-4698-9959-8707626ea7b5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.587990842Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"Crucible stats registered with UUID: f5827950-7386-4698-9959-8707626ea7b5","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.588006901Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.619521176Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430} {"msg":"f5827950-7386-4698-9959-8707626ea7b5 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.619562134Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.619592442Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"client":"0","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.619638239Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"client":"0","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.619673937Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"client":"1","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.619712425Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"client":"1","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.619744793Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"client":"2","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.619777912Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"client":"2","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.61981107Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.619841458Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.619875216Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"":"io task","client":"1","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:44162)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.619908494Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:53132)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.619942512Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:52269)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.61997654Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"":"io task","client":"1","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.620654632Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"client":"2","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.62068762Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"client":"2","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.620714968Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"client":"0","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.620743617Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"client":"0","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.620872459Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"client":"1","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.620901528Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"client":"1","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.621372861Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"client":"2","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.621412669Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"client":"0","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.621444907Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"client":"1","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.621976487Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"client":"1","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.622002335Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"client":"1","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.622016965Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.622087651Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"client":"2","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.622115029Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"client":"2","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.622130298Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.622143467Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"client":"0","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.622157447Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"client":"0","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"[0]R flush_numbers[0..12]: [101, 74, 96, 101, 101, 106, 105, 106, 103, 103, 102, 102]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.622171856Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"":"downstairs","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"[0]R generation[0..12]: [58, 44, 54, 58, 58, 61, 60, 61, 60, 60, 59, 59]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.622185705Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"":"downstairs","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.622199804Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"":"downstairs","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"[1]R flush_numbers[0..12]: [101, 74, 96, 101, 101, 106, 105, 106, 103, 103, 102, 102]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.622213623Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"":"downstairs","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"[1]R generation[0..12]: [58, 44, 54, 58, 58, 61, 60, 61, 60, 60, 59, 59]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.622231193Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"":"downstairs","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.622245452Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"":"downstairs","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"[2]R flush_numbers[0..12]: [101, 74, 96, 101, 101, 106, 105, 106, 103, 103, 102, 102]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.622259451Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"":"downstairs","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"[2]R generation[0..12]: [58, 44, 54, 58, 58, 61, 60, 61, 60, 60, 59, 59]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.62227309Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"":"downstairs","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.622287059Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"":"downstairs","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.622300089Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"":"downstairs","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.622313278Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"":"downstairs","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 107","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.622331827Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"":"downstairs","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.622345236Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"":"downstairs","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.622358675Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.622370655Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"client":"0","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.622383964Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"client":"1","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.622397033Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"client":"2","":"downstairs","session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.622410402Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.622422652Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"f5827950-7386-4698-9959-8707626ea7b5 is now active with session: 8ed46881-1de8-4681-9977-75570486b6d4","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.622438991Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.62245155Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430,"session_id":"8ed46881-1de8-4681-9977-75570486b6d4"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:20.622463959Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 156 len 40960 data: 6 6 6 7 4 4 4 4 4 5 02/30 Read block 269 len 24576 03/30 Write block 571 len 4096 data: 6 04/30 Read block 241 len 28672 05/30 Read block 164 len 8192 06/30 Write block 79 len 8192 data: 8 7 07/30 Write block 70 len 32768 data: 6 7 8 9 7 5 7 6 08/30 Write block 217 len 40960 data: 7 6 6 5 4 5 5 6 5 5 09/30 Write block 274 len 28672 data: 5 7 7 9 10 9 8 10/30 Read block 143 len 24576 11/30 Read block 536 len 28672 12/30 Write block 241 len 4096 data: 6 13/30 Read block 483 len 32768 14/30 Write block 441 len 40960 data: 4 3 4 5 6 7 6 5 7 6 15/30 Read block 267 len 32768 16/30 Write block 47 len 28672 data: 4 7 7 8 7 8 8 17/30 Read block 536 len 8192 18/30 Read block 535 len 20480 19/30 Write block 315 len 12288 data: 6 6 8 20/30 Flush 21/30 Write block 456 len 24576 data: 8 8 7 6 5 5 22/30 Read block 580 len 12288 23/30 Read block 216 len 28672 24/30 Write block 525 len 40960 data: 7 7 6 5 5 5 4 4 5 5 25/30 Write block 39 len 36864 data: 4 3 3 3 4 3 3 4 5 26/30 Flush 27/30 Write block 276 len 32768 data: 8 10 11 10 9 7 8 6 28/30 Write block 526 len 4096 data: 8 29/30 Write block 469 len 8192 data: 6 5 30/30 Read block 213 len 28672 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 8 Sent Done Sent false 28 Acked 1027 Write 1 Sent Done Sent false 29 Acked 1028 Write 2 Sent Done Sent false 30 Acked 1029 Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 0 4 8 Done 0 4 0 4 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:30:20.647393017Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1430} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 101 101 101 F F F 1 020-039 62 62 62 108 108 108 F F F 2 040-059 62 62 62 108 108 108 F F F 3 060-079 62 62 62 107 107 107 F F F 4 080-099 62 62 62 107 107 107 F F F 5 100-119 61 61 61 106 106 106 F F F 6 120-139 60 60 60 105 105 105 F F F 7 140-159 62 62 62 107 107 107 F F F 8 160-179 62 62 62 107 107 107 F F F 9 180-199 60 60 60 103 103 103 F F F 10 200-219 62 62 62 107 107 107 F F F 11 220-239 62 62 62 107 107 107 F F F 12 240-259 62 62 62 107 107 107 F F F 13 260-279 62 62 62 107 107 107 T T T 14 280-299 62 62 62 107 107 107 T T T 15 300-319 62 62 62 107 107 107 F F F 16 320-339 59 59 59 102 102 102 F F F 17 340-359 60 60 60 104 104 104 F F F 18 360-379 61 61 61 106 106 106 F F F 19 380-399 61 61 61 106 106 106 F F F 20 400-419 53 53 53 94 94 94 F F F 21 420-439 57 57 57 100 100 100 F F F 22 440-459 62 62 62 108 108 108 F F F 23 460-479 62 62 62 108 108 108 T T T 24 480-499 59 59 59 102 102 102 F F F 25 500-519 61 61 61 106 106 106 F F F 26 520-539 62 62 62 108 108 108 T T T 27 540-559 60 60 60 105 105 105 F F F 28 560-579 62 62 62 107 107 107 F F F 29 580-599 60 60 60 103 103 103 F F F Max gen: 62, Max flush: 108 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 101 101 101 F F F 1 020-039 62 62 62 108 108 108 F F F 2 040-059 62 62 62 108 108 108 F F F 3 060-079 62 62 62 107 107 107 F F F 4 080-099 62 62 62 107 107 107 F F F 5 100-119 61 61 61 106 106 106 F F F 6 120-139 60 60 60 105 105 105 F F F 7 140-159 62 62 62 107 107 107 F F F 8 160-179 62 62 62 107 107 107 F F F 9 180-199 60 60 60 103 103 103 F F F 10 200-219 62 62 62 107 107 107 F F F 11 220-239 62 62 62 107 107 107 F F F 12 240-259 62 62 62 107 107 107 F F F 13 260-279 63 63 63 109 109 109 F F F 14 280-299 63 63 63 109 109 109 F F F 15 300-319 62 62 62 107 107 107 F F F 16 320-339 59 59 59 102 102 102 F F F 17 340-359 60 60 60 104 104 104 F F F 18 360-379 61 61 61 106 106 106 F F F 19 380-399 61 61 61 106 106 106 F F F 20 400-419 53 53 53 94 94 94 F F F 21 420-439 57 57 57 100 100 100 F F F 22 440-459 62 62 62 108 108 108 F F F 23 460-479 63 63 63 109 109 109 F F F 24 480-499 59 59 59 102 102 102 F F F 25 500-519 61 61 61 106 106 106 F F F 26 520-539 63 63 63 109 109 109 F F F 27 540-559 60 60 60 105 105 105 F F F 28 560-579 62 62 62 107 107 107 F F F 29 580-599 60 60 60 103 103 103 F F F Max gen: 63, Max flush: 109 Begin loop 32 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.015818305Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.016123678Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.016159256Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.016192174Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438} {"msg":"Crucible e85c05ab-f0b3-4a67-be09-e4ba114fdeea has session id: 0945adbe-8b04-4af6-9be2-2c82047d4c4e","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.016281459Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"Upstairs opts: Upstairs UUID: e85c05ab-f0b3-4a67-be09-e4ba114fdeea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.016331246Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"Crucible stats registered with UUID: e85c05ab-f0b3-4a67-be09-e4ba114fdeea","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.016357944Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.047345709Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438} {"msg":"e85c05ab-f0b3-4a67-be09-e4ba114fdeea active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.047381858Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.047405936Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"client":"0","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.047449844Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"client":"0","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.047484352Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"client":"1","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.04751897Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"client":"1","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.047559317Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"client":"2","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.047595166Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"client":"2","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.047627324Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.047665611Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.04769976Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:60889)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.047735338Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:49342)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.047807314Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:61138)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.047845981Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.048598609Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"client":"2","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.048625877Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"client":"2","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.048642346Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"client":"1","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.048664525Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"client":"1","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.04875225Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"client":"0","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.048777479Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"client":"0","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.049386214Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"client":"1","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.049412152Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"client":"2","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.049429571Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"client":"0","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.050009129Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"client":"1","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.050040267Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"client":"1","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.050054776Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.050067215Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"client":"0","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.050080725Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"client":"0","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.050093954Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.050106043Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"client":"2","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.050119123Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"client":"2","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"[0]R flush_numbers[0..12]: [101, 108, 108, 107, 107, 106, 105, 107, 107, 103, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.050132922Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"":"downstairs","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"[0]R generation[0..12]: [58, 62, 62, 62, 62, 61, 60, 62, 62, 60, 62, 62]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.050146931Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"":"downstairs","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.05016112Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"":"downstairs","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"[1]R flush_numbers[0..12]: [101, 108, 108, 107, 107, 106, 105, 107, 107, 103, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.050176619Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"":"downstairs","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"[1]R generation[0..12]: [58, 62, 62, 62, 62, 61, 60, 62, 62, 60, 62, 62]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.050201038Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"":"downstairs","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.050228576Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"":"downstairs","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"[2]R flush_numbers[0..12]: [101, 108, 108, 107, 107, 106, 105, 107, 107, 103, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.050248425Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"":"downstairs","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"[2]R generation[0..12]: [58, 62, 62, 62, 62, 61, 60, 62, 62, 60, 62, 62]","v":0,"name":"crucible","level":30,"timeWait for a query_work_queue command to finish before sending IO ":"2024-09-26T14:30:25.050269984Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"":"downstairs","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.050303312Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"":"downstairs","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.050328001Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"":"downstairs","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.050352449Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"":"downstairs","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"Next flush: 110","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.050376998Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"":"downstairs","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.050396857Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"":"downstairs","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-09-26T14:30:25.050420786Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.050445494Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"client":"0","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.050467833Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"client":"1","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.050489522Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"client":"2","":"downstairs","session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.0505129Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.050534159Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"e85c05ab-f0b3-4a67-be09-e4ba114fdeea is now active with session: 0945adbe-8b04-4af6-9be2-2c82047d4c4e","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.050556448Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.050576477Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438,"session_id":"0945adbe-8b04-4af6-9be2-2c82047d4c4e"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:25.050596825Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 523 len 24576 data: 7 7 8 9 7 6 02/30 Write block 313 len 8192 data: 9 8 03/30 Read block 378 len 4096 04/30 Read block 64 len 28672 05/30 Write block 97 len 32768 data: 2 3 3 4 6 4 4 4 06/30 Read block 412 len 24576 07/30 Read block 24 len 28672 08/30 Flush 09/30 Write block 416 len 32768 data: 6 5 4 4 4 4 4 5 10/30 Write block 254 len 12288 data: 2 3 2 11/30 Write block 426 len 16384 data: 4 5 6 5 12/30 Write block 86 len 32768 data: 6 6 6 6 7 7 6 6 13/30 Flush 14/30 Write block 540 len 4096 data: 7 15/30 Read block 326 len 24576 16/30 Flush 17/30 Write block 546 len 12288 data: 7 6 6 18/30 Write block 302 len 36864 data: 7 10 9 11 9 10 10 9 6 19/30 Write block 197 len 12288 data: 6 7 5 20/30 Write block 181 len 20480 data: 7 9 10 10 9 21/30 Write block 176 len 8192 data: 4 3 22/30 Read block 91 len 20480 23/30 Write block 488 len 20480 data: 5 5 5 5 5 24/30 Write block 156 len 4096 data: 7 25/30 Write block 539 len 36864 data: 6 8 6 5 6 7 8 8 7 26/30 Write block 113 len 20480 data: 10 9 7 6 6 27/30 Write block 3 len 4096 data: 5 28/30 Write block 9 len 12288 data: 4 5 5 29/30 Write block 151 len 32768 data: 4 4 3 3 4 8 7 7 30/30 Read block 167 len 32768 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 17 Acked 1016 Write 3 Done Done Done false 18 Acked 1017 Write 9 Done Done Done false 19 Acked 1018 Write 3 Done Done Done false 20 Acked 1019 Write 5 Done Done Done false 21 Acked 1020 Write 2 Done Done Done false 22 Acked 1021 Read 5 Done Done Done false 23 Acked 1022 Write 5 Done Done Done false 24 Acked 1023 Write 1 Done Done Done false 25 Acked 1024 Write 9 Done Done Done false 26 Acked 1025 Write 5 Done Done Sent false 27 Acked 1026 Write 1 Done Done Sent false 28 Acked 1027 Write 3 Done Done Sent false 29 Acked 1028 Write 8 Done Done Sent false 30 Acked 1029 Read 8 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 5 6 Done 13 14 9 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:30:25.07223573Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1438} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 101 101 101 T T T 1 020-039 62 62 62 108 108 108 F F F 2 040-059 62 62 62 108 108 108 F F F 3 060-079 62 62 62 107 107 107 F F F 4 080-099 64 64 64 111 111 111 F F F 5 100-119 64 64 64 110 110 110 T T T 6 120-139 60 60 60 105 105 105 F F F 7 140-159 62 62 62 107 107 107 T T T 8 160-179 62 62 62 107 107 107 T T T 9 180-199 60 60 60 103 103 103 T T T 10 200-219 62 62 62 107 107 107 F F F 11 220-239 62 62 62 107 107 107 F F F 12 240-259 64 64 64 111 111 111 F F F 13 260-279 63 63 63 109 109 109 F F F 14 280-299 63 63 63 109 109 109 F F F 15 300-319 64 64 64 110 110 110 T T T 16 320-339 59 59 59 102 102 102 F F F 17 340-359 60 60 60 104 104 104 F F F 18 360-379 61 61 61 106 106 106 F F F 19 380-399 61 61 61 106 106 106 F F F 20 400-419 64 64 64 111 111 111 F F F 21 420-439 64 64 64 111 111 111 F F F 22 440-459 62 62 62 108 108 108 F F F 23 460-479 63 63 63 109 109 109 F F F 24 480-499 59 59 59 102 102 102 T T T 25 500-519 61 61 61 106 106 106 F F F 26 520-539 64 64 64 110 110 110 T T T 27 540-559 64 64 64 112 112 112 T T T 28 560-579 62 62 62 107 107 107 F F F 29 580-599 60 60 60 103 103 103 F F F Max gen: 64, Max flush: 112 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 113 113 113 F F F 1 020-039 62 62 62 108 108 108 F F F 2 040-059 62 62 62 108 108 108 F F F 3 060-079 62 62 62 107 107 107 F F F 4 080-099 64 64 64 111 111 111 F F F 5 100-119 65 65 65 113 113 113 F F F 6 120-139 60 60 60 105 105 105 F F F 7 140-159 65 65 65 113 113 113 F F F 8 160-179 65 65 65 113 113 113 F F F 9 180-199 65 65 65 113 113 113 F F F 10 200-219 62 62 62 107 107 107 F F F 11 220-239 62 62 62 107 107 107 F F F 12 240-259 64 64 64 111 111 111 F F F 13 260-279 63 63 63 109 109 109 F F F 14 280-299 63 63 63 109 109 109 F F F 15 300-319 65 65 65 113 113 113 F F F 16 320-339 59 59 59 102 102 102 F F F 17 340-359 60 60 60 104 104 104 F F F 18 360-379 61 61 61 106 106 106 F F F 19 380-399 61 61 61 106 106 106 F F F 20 400-419 64 64 64 111 111 111 F F F 21 420-439 64 64 64 111 111 111 F F F 22 440-459 62 62 62 108 108 108 F F F 23 460-479 63 63 63 109 109 109 F F F 24 480-499 65 65 65 113 113 113 F F F 25 500-519 61 61 61 106 106 106 F F F 26 520-539 65 65 65 113 113 113 F F F 27 540-559 65 65 65 113 113 113 F F F 28 560-579 62 62 62 107 107 107 F F F 29 580-599 60 60 60 103 103 103 F F F Max gen: 65, Max flush: 113 Begin loop 33 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.531245337Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.531559029Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.531592797Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.531623975Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446} {"msg":"Crucible a1fafd89-dee6-45ea-8254-b86a8a422026 has session id: ba67ca8f-7404-45c6-abfb-814832c3946c","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.531657553Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"Upstairs opts: Upstairs UUID: a1fafd89-dee6-45ea-8254-b86a8a422026, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.5317073Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"Crucible stats registered with UUID: a1fafd89-dee6-45ea-8254-b86a8a422026","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.531735399Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.562866796Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446} {"msg":"a1fafd89-dee6-45ea-8254-b86a8a422026 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.562892534Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.562907793Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"client":"0","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.562927132Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"client":"0","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.562941102Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"client":"1","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.562954451Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"client":"1","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.56296764Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"client":"2","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.562981249Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"client":"2","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.562994359Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.563008428Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.563022247Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"ds_connection connected from Ok(127.0.0.1:35121)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.563180448Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"ds_connection connected from Ok(127.0.0.1:35055)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.563239355Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"ds_connection connected from Ok(127.0.0.1:56041)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.563391506Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.563901307Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"client":"1","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.563934255Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"client":"1","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.563953834Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"client":"2","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.563971253Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"client":"2","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.564046759Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"client":"0","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.564079127Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"client":"0","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.564504793Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"client":"2","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.564683233Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"client":"1","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.564760288Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"client":"0","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.565098109Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"client":"2","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.565124628Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"client":"2","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.565138757Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.565237812Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"client":"1","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.56527415Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"client":"1","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.565289918Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.565366074Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"client":"0","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.565385603Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"client":"0","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"[0]R flush_numbers[0..12]: [113, 108, 108, 107, 111, 113, 105, 113, 113, 113, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.565399942Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"":"downstairs","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"[0]R generation[0..12]: [65, 62, 62, 62, 64, 65, 60, 65, 65, 65, 62, 62]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.565413831Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"":"downstairs","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.565428041Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"":"downstairs","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"[1]R flush_numbers[0..12]: [113, 108, 108, 107, 111, 113, 105, 113, 113, 113, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.56544211Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"":"downstairs","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"[1]R generation[0..12]: [65, 62, 62, 62, 64, 65, 60, 65, 65, 65, 62, 62]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.565455669Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"":"downstairs","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.565470878Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"":"downstairs","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"[2]R flush_numbers[0..12]: [113, 108, 108, 107, 111, 113, 105, 113, 113, 113, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.565484747Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"":"downstairs","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"[2]R generation[0..12]: [65, 62, 62, 62, 64, 65, 60, 65, 65, 65, 62, 62]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.565498217Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"":"downstairs","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.565511906Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"":"downstairs","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.565524725Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"":"downstairs","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.565540734Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"":"downstairs","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"Next flush: 114","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.565553954Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"":"downstairs","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.565567053Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"":"downstairs","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-26T14:30:29.565580112Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.565595441Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"client":"0","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.565608891Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"client":"1","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.56562193Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"client":"2","":"downstairs","session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.565635239Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.565647558Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"a1fafd89-dee6-45ea-8254-b86a8a422026 is now active with session: ba67ca8f-7404-45c6-abfb-814832c3946c","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.565660357Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.565672587Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446,"session_id":"ba67ca8f-7404-45c6-abfb-814832c3946c"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:29.565684826Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 397 len 12288 data: 8 9 8 02/30 Read block 459 len 16384 03/30 Read block 127 len 8192 04/30 Read block 531 len 40960 05/30 Read block 385 len 12288 06/30 Write block 103 len 36864 data: 5 5 4 3 5 7 6 5 8 07/30 Write block 145 len 12288 data: 3 3 4 08/30 Write block 213 len 28672 data: 6 6 5 6 8 7 7 09/30 Read block 260 len 36864 10/30 Read block 324 len 28672 11/30 Read block 239 len 16384 12/30 Read block 229 len 28672 13/30 Read block 229 len 36864 14/30 Read block 467 len 24576 15/30 Read block 405 len 24576 16/30 Write block 50 len 16384 data: 9 8 9 9 17/30 Write block 328 len 16384 data: 7 6 5 6 18/30 Read block 173 len 16384 19/30 Write block 471 len 24576 data: 5 3 3 2 3 3 20/30 Flush 21/30 Write block 230 len 4096 data: 6 22/30 Read block 387 len 12288 23/30 Write block 433 len 12288 data: 6 5 5 24/30 Read block 98 len 12288 25/30 Read block 445 len 32768 26/30 Write block 436 len 36864 data: 4 4 4 3 3 5 4 5 6 27/30 Read block 277 len 4096 28/30 Write block 128 len 24576 data: 4 5 5 4 5 5 29/30 Read block 337 len 24576 30/30 Write block 493 len 28672 data: 6 7 7 7 7 8 9 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 1 Done Done Done false 22 Acked 1021 Read 3 Done Done Done false 23 Acked 1022 Write 3 Done Done Done false 24 Acked 1023 Read 3 Done Done Done false 25 Acked 1024 Read 8 Done Done Done false 26 Acked 1025 Write 9 Done Done Done false 27 Acked 1026 Read 1 Done Done Done false 28 Acked 1027 Write 6 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:30:29.58840432Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1446} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 113 113 113 F F F 1 020-039 62 62 62 108 108 108 F F F 2 040-059 66 66 66 114 114 114 F F F 3 060-079 62 62 62 107 107 107 F F F 4 080-099 64 64 64 111 111 111 F F F 5 100-119 66 66 66 114 114 114 F F F 6 120-139 60 60 60 105 105 105 T T T 7 140-159 66 66 66 114 114 114 F F F 8 160-179 65 65 65 113 113 113 F F F 9 180-199 65 65 65 113 113 113 F F F 10 200-219 66 66 66 114 114 114 F F F 11 220-239 62 62 62 107 107 107 T T T 12 240-259 64 64 64 111 111 111 F F F 13 260-279 63 63 63 109 109 109 F F F 14 280-299 63 63 63 109 109 109 F F F 15 300-319 65 65 65 113 113 113 F F F 16 320-339 66 66 66 114 114 114 F F F 17 340-359 60 60 60 104 104 104 F F F 18 360-379 61 61 61 106 106 106 F F F 19 380-399 66 66 66 114 114 114 F F F 20 400-419 64 64 64 111 111 111 F F F 21 420-439 64 64 64 111 111 111 T T T 22 440-459 62 62 62 108 108 108 T T T 23 460-479 66 66 66 114 114 114 F F F 24 480-499 65 65 65 113 113 113 T T T 25 500-519 61 61 61 106 106 106 F F F 26 520-539 65 65 65 113 113 113 F F F 27 540-559 65 65 65 113 113 113 F F F 28 560-579 62 62 62 107 107 107 F F F 29 580-599 60 60 60 103 103 103 F F F Max gen: 66, Max flush: 114 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 113 113 113 F F F 1 020-039 62 62 62 108 108 108 F F F 2 040-059 66 66 66 114 114 114 F F F 3 060-079 62 62 62 107 107 107 F F F 4 080-099 64 64 64 111 111 111 F F F 5 100-119 66 66 66 114 114 114 F F F 6 120-139 67 67 67 115 115 115 F F F 7 140-159 66 66 66 114 114 114 F F F 8 160-179 65 65 65 113 113 113 F F F 9 180-199 65 65 65 113 113 113 F F F 10 200-219 66 66 66 114 114 114 F F F 11 220-239 67 67 67 115 115 115 F F F 12 240-259 64 64 64 111 111 111 F F F 13 260-279 63 63 63 109 109 109 F F F 14 280-299 63 63 63 109 109 109 F F F 15 300-319 65 65 65 113 113 113 F F F 16 320-339 66 66 66 114 114 114 F F F 17 340-359 60 60 60 104 104 104 F F F 18 360-379 61 61 61 106 106 106 F F F 19 380-399 66 66 66 114 114 114 F F F 20 400-419 64 64 64 111 111 111 F F F 21 420-439 67 67 67 115 115 115 F F F 22 440-459 67 67 67 115 115 115 F F F 23 460-479 66 66 66 114 114 114 F F F 24 480-499 67 67 67 115 115 115 F F F 25 500-519 61 61 61 106 106 106 F F F 26 520-539 65 65 65 113 113 113 F F F 27 540-559 65 65 65 113 113 113 F F F 28 560-579 62 62 62 107 107 107 F F F 29 580-599 60 60 60 103 103 103 F F F Max gen: 67, Max flush: 115 Begin loop 34 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:33.997797167Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:33.998138898Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:33.998161436Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:33.998187025Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454} {"msg":"Crucible 083aae6f-c314-41ff-970f-da78845e3419 has session id: ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:33.998322767Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"Upstairs opts: Upstairs UUID: 083aae6f-c314-41ff-970f-da78845e3419, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:33.998375044Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"Crucible stats registered with UUID: 083aae6f-c314-41ff-970f-da78845e3419","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:33.998404522Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.03022607Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454} {"msg":"083aae6f-c314-41ff-970f-da78845e3419 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.030269098Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.030286797Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"client":"0","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.030311195Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"client":"0","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.030328874Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"client":"1","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.030345894Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"client":"1","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.030362782Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"client":"2","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.030379492Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"client":"2","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.03039612Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.030419179Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.030448728Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:57174)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.030798138Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:47138)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.030841285Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:41082)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.030865594Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.031314248Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"client":"2","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.031345817Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"client":"2","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.031362866Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"client":"1","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.031377235Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"client":"1","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.031492089Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"client":"0","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.031530706Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"client":"0","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.032027078Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"client":"1","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.032058866Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"client":"0","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.032104374Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"client":"2","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.032652083Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"client":"1","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.032679301Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"client":"1","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.03269383Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.03270641Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"client":"0","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.032719869Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"client":"0","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.032733108Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.032746458Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"client":"2","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.032763357Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"client":"2","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"[0]R flush_numbers[0..12]: [113, 108, 114, 107, 111, 114, 115, 114, 113, 113, 114, 115]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.032777946Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"":"downstairs","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"[0]R generation[0..12]: [65, 62, 66, 62, 64, 66, 67, 66, 65, 65, 66, 67]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.032792175Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"":"downstairs","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.032806924Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"":"downstairs","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"[1]R flush_numbers[0..12]: [113, 108, 114, 107, 111, 114, 115, 114, 113, 113, 114, 115]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.032821953Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"":"downstairs","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"[1]R generation[0..12]: [65, 62, 66, 62, 64, 66, 67, 66, 65, 65, 66, 67]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.032835693Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"":"downstairs","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.032849882Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"":"downstairs","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"[2]R flush_numbers[0..12]: [113, 108, 114, 107, 111, 114, 115, 114, 113, 113, 114, 115]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.032863711Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"":"downstairs","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"[2]R generation[0..12]: [65, 62, 66, 62, 64, 66, 67, 66, 65, 65, 66, 67]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.03287734Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"":"downstairs","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.032891179Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"":"downstairs","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.032918168Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"":"downstairs","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.032931927Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"":"downstairs","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"Next flush: 116","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.032944906Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"":"downstairs","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.032957775Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"":"downstairs","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"Wait for a query_work_queue command to finish before sending IO No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.032978254Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.032991044Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"client":"0","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.033004273Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"client":"1","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.033017512Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"client":"2","":"downstairs","session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.033030731Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.033042901Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"083aae6f-c314-41ff-970f-da78845e3419 is now active with session: ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.03305574Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.033067829Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454,"session_id":"ab74ff4b-a16f-4997-ae26-ad7f8f5c1c8d"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:34.033080239Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 155 len 20480 data: 5 9 8 8 8 02/30 Write block 295 len 40960 data: 9 8 5 9 8 7 4 8 11 10 03/30 Write block 447 len 4096 data: 7 04/30 Write block 84 len 16384 data: 3 3 7 7 05/30 Write block 85 len 24576 data: 4 8 8 7 7 8 06/30 Flush 07/30 Write block 221 len 4096 data: 5 08/30 Read block 547 len 12288 09/30 Write block 492 len 24576 data: 6 7 8 8 8 8 10/30 Read block 5 len 32768 11/30 Flush 12/30 Write block 247 len 16384 data: 7 5 5 3 13/30 Read block 133 len 20480 14/30 Read block 207 len 12288 15/30 Write block 35 len 24576 data: 4 4 4 4 5 4 16/30 Write block 531 len 12288 data: 5 5 6 17/30 Read block 232 len 28672 18/30 Write block 421 len 36864 data: 5 5 6 4 4 5 6 7 6 19/30 Read block 228 len 32768 20/30 Write block 216 len 40960 data: 7 9 8 8 6 6 6 6 7 6 21/30 Flush 22/30 Write block 319 len 12288 data: 7 6 5 23/30 Flush 24/30 Write block 252 len 20480 data: 3 2 3 4 3 25/30 Read block 95 len 28672 26/30 Read block 246 len 20480 27/30 Write block 99 len 32768 data: 4 5 7 5 6 6 5 4 28/30 Read block 141 len 28672 29/30 Write block 275 len 36864 data: 8 9 11 12 11 10 8 9 7 30/30 Read block 40 len 32768 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 5 Done Done Done false 25 Acked 1024 Read 7 Done Done Done false 26 Acked 1025 Read 5 Done Done Done false 27 Acked 1026 Write 8 Done Done Done false 28 Acked 1027 Read 7 Done Done Done false 29 Acked 1028 Write 9 Done Sent Sent false 30 Acked 1029 Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 7 5 5 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:30:34.06183764Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1454} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 113 113 113 F F F 1 020-039 68 68 68 118 118 118 F F F 2 040-059 68 68 68 118 118 118 F F F 3 060-079 62 62 62 107 107 107 F F F 4 080-099 68 68 68 116 116 116 T T T 5 100-119 66 66 66 114 114 114 T T T 6 120-139 67 67 67 115 115 115 F F F 7 140-159 68 68 68 116 116 116 F F F 8 160-179 65 65 65 113 113 113 F F F 9 180-199 65 65 65 113 113 113 F F F 10 200-219 68 68 68 118 118 118 F F F 11 220-239 68 68 68 118 118 118 F F F 12 240-259 68 68 68 118 118 118 T T T 13 260-279 63 63 63 109 109 109 T T T 14 280-299 68 68 68 116 116 116 T T T 15 300-319 68 68 68 119 119 119 F F F 16 320-339 68 68 68 119 119 119 F F F 17 340-359 60 60 60 104 104 104 F F F 18 360-379 61 61 61 106 106 106 F F F 19 380-399 66 66 66 114 114 114 F F F 20 400-419 64 64 64 111 111 111 F F F 21 420-439 68 68 68 118 118 118 F F F 22 440-459 68 68 68 116 116 116 F F F 23 460-479 66 66 66 114 114 114 F F F 24 480-499 68 68 68 117 117 117 F F F 25 500-519 61 61 61 106 106 106 F F F 26 520-539 68 68 68 118 118 118 F F F 27 540-559 65 65 65 113 113 113 F F F 28 560-579 62 62 62 107 107 107 F F F 29 580-599 60 60 60 103 103 103 F F F Max gen: 68, Max flush: 119 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 113 113 113 F F F 1 020-039 68 68 68 118 118 118 F F F 2 040-059 68 68 68 118 118 118 F F F 3 060-079 62 62 62 107 107 107 F F F 4 080-099 69 69 69 120 120 120 F F F 5 100-119 69 69 69 120 120 120 F F F 6 120-139 67 67 67 115 115 115 F F F 7 140-159 68 68 68 116 116 116 F F F 8 160-179 65 65 65 113 113 113 F F F 9 180-199 65 65 65 113 113 113 F F F 10 200-219 68 68 68 118 118 118 F F F 11 220-239 68 68 68 118 118 118 F F F 12 240-259 69 69 69 120 120 120 F F F 13 260-279 69 69 69 120 120 120 F F F 14 280-299 69 69 69 120 120 120 F F F 15 300-319 68 68 68 119 119 119 F F F 16 320-339 68 68 68 119 119 119 F F F 17 340-359 60 60 60 104 104 104 F F F 18 360-379 61 61 61 106 106 106 F F F 19 380-399 66 66 66 114 114 114 F F F 20 400-419 64 64 64 111 111 111 F F F 21 420-439 68 68 68 118 118 118 F F F 22 440-459 68 68 68 116 116 116 F F F 23 460-479 66 66 66 114 114 114 F F F 24 480-499 68 68 68 117 117 117 F F F 25 500-519 61 61 61 106 106 106 F F F 26 520-539 68 68 68 118 118 118 F F F 27 540-559 65 65 65 113 113 113 F F F 28 560-579 62 62 62 107 107 107 F F F 29 580-599 60 60 60 103 103 103 F F F Max gen: 69, Max flush: 120 Begin loop 35 Downstairs to restart: 2 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.456211468Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.456493412Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.456517641Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.456554159Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462} {"msg":"Crucible 715d0801-0848-48f5-87b2-a61fad99402b has session id: 729c101a-9cf0-40f8-bdc3-5dd54b7903eb","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.456595556Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"Upstairs opts: Upstairs UUID: 715d0801-0848-48f5-87b2-a61fad99402b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.456636554Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"Crucible stats registered with UUID: 715d0801-0848-48f5-87b2-a61fad99402b","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.456659533Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.487915422Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462} {"msg":"715d0801-0848-48f5-87b2-a61fad99402b active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.48795028Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.487974389Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"client":"0","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.487997788Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"client":"0","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.488020586Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"client":"1","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.488037625Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"client":"1","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.488054204Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"client":"2","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.488071134Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"client":"2","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.488087773Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"":"io task","client":"2","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.488104902Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.488121741Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:43517)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.488219745Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"":"io task","client":"2","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:58939)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.488266252Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:45960)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.488300541Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.489064147Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"client":"0","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.489097076Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"client":"0","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.489125804Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"client":"1","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.489153452Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"client":"1","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.489220529Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"client":"2","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.489250057Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"client":"2","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.489700031Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"client":"1","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.489739169Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"client":"2","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.489861802Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"client":"0","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.490304567Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"client":"1","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.490342755Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"client":"1","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.490370423Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.490459168Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"client":"2","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.490487637Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"client":"2","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.490503736Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.490516695Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"client":"0","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.490529984Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"client":"0","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"[0]R flush_numbers[0..12]: [113, 118, 118, 107, 120, 120, 115, 116, 113, 113, 118, 118]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.490543954Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"":"downstairs","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"[0]R generation[0..12]: [65, 68, 68, 62, 69, 69, 67, 68, 65, 65, 68, 68]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.490557743Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"":"downstairs","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.490572062Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"":"downstairs","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"[1]R flush_numbers[0..12]: [113, 118, 118, 107, 120, 120, 115, 116, 113, 113, 118, 118]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.490586151Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"":"downstairs","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"[1]R generation[0..12]: [65, 68, 68, 62, 69, 69, 67, 68, 65, 65, 68, 68]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.4906037Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"":"downstairs","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.490618309Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"":"downstairs","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"[2]R flush_numbers[0..12]: [113, 118, 118, 107, 120, 120, 115, 116, 113, 113, 118, 118]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.490632029Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"":"downstairs","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"[2]R generation[0..12]: [65, 68, 68, 62, 69, 69, 67, 68, 65, 65, 68, 68]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.490645528Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"":"downstairs","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.490659517Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"":"downstairs","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.490672396Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"":"downstairs","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.490685375Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"":"downstairs","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"Next flush: 121","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.490703045Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"":"downstairs","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.490730783Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"":"downstairs","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.490752152Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.490774421Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"client":"0","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.490800729Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"client":"1","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.490826287Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"client":"2","":"downstairs","session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.490852376Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.490876705Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"715d0801-0848-48f5-87b2-a61fad99402b is now active with session: 729c101a-9cf0-40f8-bdc3-5dd54b7903eb","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.490906943Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.490937181Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462,"session_id":"729c101a-9cf0-40f8-bdc3-5dd54b7903eb"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:38.49096149Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 389 len 24576 data: 2 3 3 3 3 4 02/30 Flush 03/30 Write block 251 len 24576 data: 3 4 3 4 5 4 04/30 Read block 162 len 40960 05/30 Read block 155 len 36864 06/30 Write block 17 len 16384 data: 6 6 5 3 07/30 Read block 46 len 8192 08/30 Read block 344 len 16384 09/30 Read block 281 len 24576 10/30 Read block 172 len 4096 11/30 Read block 16 len 24576 12/30 Write block 169 len 12288 data: 4 6 5 13/30 Write block 109 len 20480 data: 7 6 9 10 11 14/30 Write block 311 len 24576 data: 7 7 10 9 7 7 15/30 Write block 79 len 32768 data: 9 8 8 7 4 4 5 9 16/30 Read block 324 len 4096 17/30 Read block 333 len 12288 18/30 Read block 498 len 24576 19/30 Write block 548 len 24576 data: 7 5 6 5 4 4 20/30 Read block 258 len 4096 21/30 Read block 107 len 40960 22/30 Write block 156 len 36864 data: 10 9 9 9 5 5 5 5 5 23/30 Write block 523 len 8192 data: 8 8 24/30 Read block 330 len 20480 25/30 Read block 123 len 40960 26/30 Read block 252 len 24576 27/30 Read block 569 len 32768 28/30 Read block 401 len 36864 29/30 Write block 100 len 36864 data: 6 8 6 7 7 6 5 6 8 30/30 Read block 172 len 20480 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 3 Acked 1002 Write 6 Done Done Done false 4 Acked 1003 Read 10 Done Done Done false 5 Acked 1004 Read 9 Done Done Done false 6 Acked 1005 Write 4 Done Done Done false 7 Acked 1006 Read 2 Done Done Done false 8 Acked 1007 Read 4 Done Done Done false 9 Acked 1008 Read 6 Done Done Done false 10 Acked 1009 Read 1 Done Done Done false 11 Acked 1010 Read 6 Done Done Done false 12 Acked 1011 Write 3 Done Done Done false 13 Acked 1012 Write 5 Done Done Done false 14 Acked 1013 Write 6 Done Done Done false 15 Acked 1014 Write 8 Done Done Done false 16 Acked 1015 Read 1 Done Done Done false 17 Acked 1016 Read 3 Done Done Done false 18 Acked 1017 Read 6 Done Done Done false 19 Acked 1018 Write 6 Done Done Done false 20 Acked 1019 Read 1 Done Done Done false 21 Acked 1020 Read 10 Done Done Done false 22 Acked 1021 Write 9 Done Done Done false 23 Acked 1022 Write 2 Done Done Done false 24 Acked 1023 Read 5 Done Done Done false 25 Acked 1024 Read 10 Done Done Done false 26 Acked 1025 Read 6 Done Done Done false 27 Acked 1026 Read 8 Done Done Done false 28 Acked 1027 Read 9 Done Done Done false 29 Acked 1028 Write 9 Done Done Sent false 30 Acked 1029 Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 28 27 26 81 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1001 1001 1001 Downstairs last five completed: 1001 1000 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:30:38.513584869Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1462} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 113 113 113 T T T 1 020-039 68 68 68 118 118 118 T T T 2 040-059 68 68 68 118 118 118 F F F 3 060-079 62 62 62 107 107 107 T T T 4 080-099 69 69 69 120 120 120 T T T 5 100-119 69 69 69 120 120 120 T T T 6 120-139 67 67 67 115 115 115 F F F 7 140-159 68 68 68 116 116 116 T T T 8 160-179 65 65 65 113 113 113 T T T 9 180-199 65 65 65 113 113 113 F F F 10 200-219 68 68 68 118 118 118 F F F 11 220-239 68 68 68 118 118 118 F F F 12 240-259 69 69 69 120 120 120 T T T 13 260-279 69 69 69 120 120 120 F F F 14 280-299 69 69 69 120 120 120 F F F 15 300-319 68 68 68 119 119 119 T T T 16 320-339 68 68 68 119 119 119 F F F 17 340-359 60 60 60 104 104 104 F F F 18 360-379 61 61 61 106 106 106 F F F 19 380-399 70 70 70 121 121 121 F F F 20 400-419 64 64 64 111 111 111 F F F 21 420-439 68 68 68 118 118 118 F F F 22 440-459 68 68 68 116 116 116 F F F 23 460-479 66 66 66 114 114 114 F F F 24 480-499 68 68 68 117 117 117 F F F 25 500-519 61 61 61 106 106 106 F F F 26 520-539 68 68 68 118 118 118 T T T 27 540-559 65 65 65 113 113 113 T T T 28 560-579 62 62 62 107 107 107 F F F 29 580-599 60 60 60 103 103 103 F F F Max gen: 70, Max flush: 121 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 122 122 122 F F F 1 020-039 71 71 71 122 122 122 F F F 2 040-059 68 68 68 118 118 118 F F F 3 060-079 71 71 71 122 122 122 F F F 4 080-099 71 71 71 122 122 122 F F F 5 100-119 71 71 71 122 122 122 F F F 6 120-139 67 67 67 115 115 115 F F F 7 140-159 71 71 71 122 122 122 F F F 8 160-179 71 71 71 122 122 122 F F F 9 180-199 65 65 65 113 113 113 F F F 10 200-219 68 68 68 118 118 118 F F F 11 220-239 68 68 68 118 118 118 F F F 12 240-259 71 71 71 122 122 122 F F F 13 260-279 69 69 69 120 120 120 F F F 14 280-299 69 69 69 120 120 120 F F F 15 300-319 71 71 71 122 122 122 F F F 16 320-339 68 68 68 119 119 119 F F F 17 340-359 60 60 60 104 104 104 F F F 18 360-379 61 61 61 106 106 106 F F F 19 380-399 70 70 70 121 121 121 F F F 20 400-419 64 64 64 111 111 111 F F F 21 420-439 68 68 68 118 118 118 F F F 22 440-459 68 68 68 116 116 116 F F F 23 460-479 66 66 66 114 114 114 F F F 24 480-499 68 68 68 117 117 117 F F F 25 500-519 61 61 61 106 106 106 F F F 26 520-539 71 71 71 122 122 122 F F F 27 540-559 71 71 71 122 122 122 F F F 28 560-579 62 62 62 107 107 107 F F F 29 580-599 60 60 60 103 103 103 F F F Max gen: 71, Max flush: 122 Begin loop 36 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:42.976581319Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:42.976887812Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:42.976907261Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:42.97692443Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470} {"msg":"Crucible 6a3594cf-ce41-45e0-bc34-ac14e2b3f597 has session id: 31aa44f8-b48d-417a-82b9-4b8d213e2d6a","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:42.977086641Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"Upstairs opts: Upstairs UUID: 6a3594cf-ce41-45e0-bc34-ac14e2b3f597, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:42.977131068Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"Crucible stats registered with UUID: 6a3594cf-ce41-45e0-bc34-ac14e2b3f597","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:42.977150717Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.008811034Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470} {"msg":"6a3594cf-ce41-45e0-bc34-ac14e2b3f597 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.008850972Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.00887779Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"client":"0","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.008922798Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"client":"0","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.008962875Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"client":"1","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.008999853Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"client":"1","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.009031721Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"client":"2","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.009065859Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"client":"2","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.009099128Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"":"io task","client":"2","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.009134576Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.009168504Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"":"io task","client":"1","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"ds_connection connected from Ok(127.0.0.1:46798)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.009304946Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"":"io task","client":"2","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"ds_connection connected from Ok(127.0.0.1:38227)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.009354473Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"":"io task","client":"1","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"ds_connection connected from Ok(127.0.0.1:47584)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.009387991Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.009843575Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"client":"1","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.009884463Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"client":"1","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.009913542Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"client":"2","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.009975118Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"client":"2","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.010042204Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"client":"0","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.010080172Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"client":"0","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.010451911Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"client":"1","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.010481559Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"client":"2","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.010766453Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"client":"0","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.011052437Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"client":"2","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.011081336Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"client":"2","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.011096415Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.011157651Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"client":"1","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.01117692Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"client":"1","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.011190669Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.011295283Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"client":"0","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.011314462Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"client":"0","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"[0]R flush_numbers[0..12]: [122, 122, 118, 122, 122, 122, 115, 122, 122, 113, 118, 118]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.011328731Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"":"downstairs","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"[0]R generation[0..12]: [71, 71, 68, 71, 71, 71, 67, 71, 71, 65, 68, 68]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.011343071Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"":"downstairs","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.01135776Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"":"downstairs","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"[1]R flush_numbers[0..12]: [122, 122, 118, 122, 122, 122, 115, 122, 122, 113, 118, 118]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.011375329Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"":"downstairs","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"[1]R generation[0..12]: [71, 71, 68, 71, 71, 71, 67, 71, 71, 65, 68, 68]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.011389378Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"":"downstairs","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.011404397Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"":"downstairs","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"[2]R flush_numbers[0..12]: [122, 122, 118, 122, 122, 122, 115, 122, 122, 113, 118, 118]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.011418246Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"":"downstairs","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"[2]R generation[0..12]: [71, 71, 68, 71, 71, 71, 67, 71, 71, 65, 68, 68]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.011431885Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"":"downstairs","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.011445735Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"":"downstairs","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.011458794Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"":"downstairs","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.011471843Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"":"downstairs","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"Next flush: 123","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.011484922Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"":"downstairs","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.011497932Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"":"downstairs","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"No downstairs reconciliation required"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.011514391Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.01152657Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"client":"0","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.01153975Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"client":"1","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.011552949Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"client":"2","":"downstairs","session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.011566248Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.011580967Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"6a3594cf-ce41-45e0-bc34-ac14e2b3f597 is now active with session: 31aa44f8-b48d-417a-82b9-4b8d213e2d6a","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.011593946Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.011605906Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470,"session_id":"31aa44f8-b48d-417a-82b9-4b8d213e2d6a"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:43.011618095Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 22 len 12288 data: 5 7 8 02/30 Read block 573 len 12288 03/30 Read block 572 len 8192 04/30 Read block 505 len 4096 05/30 Write block 586 len 8192 data: 3 3 06/30 Write block 267 len 40960 data: 6 6 5 4 3 3 3 6 9 10 07/30 Write block 529 len 28672 data: 6 6 6 6 7 6 5 08/30 Flush 09/30 Write block 215 len 32768 data: 6 8 10 9 9 7 7 7 10/30 Write block 397 len 4096 data: 9 11/30 Write block 578 len 40960 data: 9 6 5 5 6 7 5 5 4 4 12/30 Read block 301 len 8192 13/30 Flush 14/30 Write block 476 len 4096 data: 4 15/30 Read block 392 len 36864 16/30 Read block 275 len 16384 17/30 Read block 347 len 28672 18/30 Write block 260 len 4096 data: 5 19/30 Write block 430 len 28672 data: 6 6 6 7 6 6 5 20/30 Write block 544 len 12288 data: 8 9 9 21/30 Write block 160 len 40960 data: 6 6 6 6 6 6 4 5 5 5 22/30 Read block 230 len 36864 23/30 Read block 362 len 32768 24/30 Read block 424 len 4096 25/30 Read block 346 len 28672 26/30 Write block 527 len 8192 data: 8 7 27/30 Read block 392 len 4096 28/30 Read block 0 len 40960 29/30 Read block 451 len 4096 30/30 Read block 362 len 24576 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 14 Acked 1013 Write 1 Done Done Done false 15 Acked 1014 Read 9 Done Done Done false 16 Acked 1015 Read 4 Done Done Done false 17 Acked 1016 Read 7 Done Done Done false 18 Acked 1017 Write 1 Done Done Done false 19 Acked 1018 Write 7 Done Done Done false 20 Acked 1019 Write 3 Done Done Done false 21 Acked 1020 Write 10 Done Done Done false 22 Acked 1021 Read 9 Done Done Done false 23 Acked 1022 Read 8 Done Done Done false 24 Acked 1023 Read 1 Done Done Done false 25 Acked 1024 Read 7 Done Done Done false 26 Acked 1025 Write 2 Done Done Done false 27 Acked 1026 Read 1 Done Done Done false 28 Acked 1027 Read 10 Done Done Done false 29 Acked 1028 Read 1 Done Done Done false 30 Acked 1029 Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 17 16 16 49 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:30:43.03412513Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1470} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 122 122 122 F F F 1 020-039 72 72 72 123 123 123 F F F 2 040-059 68 68 68 118 118 118 F F F 3 060-079 71 71 71 122 122 122 F F F 4 080-099 71 71 71 122 122 122 F F F 5 100-119 71 71 71 122 122 122 F F F 6 120-139 67 67 67 115 115 115 F F F 7 140-159 71 71 71 122 122 122 F F F 8 160-179 71 71 71 122 122 122 T T T 9 180-199 65 65 65 113 113 113 F F F 10 200-219 72 72 72 124 124 124 F F F 11 220-239 72 72 72 124 124 124 F F F 12 240-259 71 71 71 122 122 122 F F F 13 260-279 72 72 72 123 123 123 T T T 14 280-299 69 69 69 120 120 120 F F F 15 300-319 71 71 71 122 122 122 F F F 16 320-339 68 68 68 119 119 119 F F F 17 340-359 60 60 60 104 104 104 F F F 18 360-379 61 61 61 106 106 106 F F F 19 380-399 72 72 72 124 124 124 F F F 20 400-419 64 64 64 111 111 111 F F F 21 420-439 68 68 68 118 118 118 T T T 22 440-459 68 68 68 116 116 116 F F F 23 460-479 66 66 66 114 114 114 T T T 24 480-499 68 68 68 117 117 117 F F F 25 500-519 61 61 61 106 106 106 F F F 26 520-539 72 72 72 123 123 123 T T T 27 540-559 71 71 71 122 122 122 T T T 28 560-579 72 72 72 124 124 124 F F F 29 580-599 72 72 72 124 124 124 F F F Max gen: 72, Max flush: 124 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 122 122 122 F F F 1 020-039 72 72 72 123 123 123 F F F 2 040-059 68 68 68 118 118 118 F F F 3 060-079 71 71 71 122 122 122 F F F 4 080-099 71 71 71 122 122 122 F F F 5 100-119 71 71 71 122 122 122 F F F 6 120-139 67 67 67 115 115 115 F F F 7 140-159 71 71 71 122 122 122 F F F 8 160-179 73 73 73 125 125 125 F F F 9 180-199 65 65 65 113 113 113 F F F 10 200-219 72 72 72 124 124 124 F F F 11 220-239 72 72 72 124 124 124 F F F 12 240-259 71 71 71 122 122 122 F F F 13 260-279 73 73 73 125 125 125 F F F 14 280-299 69 69 69 120 120 120 F F F 15 300-319 71 71 71 122 122 122 F F F 16 320-339 68 68 68 119 119 119 F F F 17 340-359 60 60 60 104 104 104 F F F 18 360-379 61 61 61 106 106 106 F F F 19 380-399 72 72 72 124 124 124 F F F 20 400-419 64 64 64 111 111 111 F F F 21 420-439 73 73 73 125 125 125 F F F 22 440-459 68 68 68 116 116 116 F F F 23 460-479 73 73 73 125 125 125 F F F 24 480-499 68 68 68 117 117 117 F F F 25 500-519 61 61 61 106 106 106 F F F 26 520-539 73 73 73 125 125 125 F F F 27 540-559 73 73 73 125 125 125 F F F 28 560-579 72 72 72 124 124 124 F F F 29 580-599 72 72 72 124 124 124 F F F Max gen: 73, Max flush: 125 Begin loop 37 Downstairs to restart: 2 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.427595219Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.427885593Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.427911961Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.42793867Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478} {"msg":"Crucible 74f5eecb-2dff-45ba-a5e9-ca6bae5393ea has session id: 7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.4281071Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"Upstairs opts: Upstairs UUID: 74f5eecb-2dff-45ba-a5e9-ca6bae5393ea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.428160417Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"Crucible stats registered with UUID: 74f5eecb-2dff-45ba-a5e9-ca6bae5393ea","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.428182596Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.459229238Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478} {"msg":"74f5eecb-2dff-45ba-a5e9-ca6bae5393ea active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.459260056Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.459280215Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"client":"0","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.459308223Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"client":"0","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.459325922Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"client":"1","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.459342941Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"client":"1","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.45935999Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"client":"2","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.459380809Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"client":"2","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.459406028Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.459437616Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.459469334Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"ds_connection connected from Ok(127.0.0.1:60136)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.459503852Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"ds_connection connected from Ok(127.0.0.1:64112)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.45953722Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"ds_connection connected from Ok(127.0.0.1:50885)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.459568619Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.460369523Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"client":"0","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.460408781Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"client":"0","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.460437479Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"client":"1","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.460465598Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"client":"1","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.460493516Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"client":"2","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.460522834Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"client":"2","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.461020476Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"client":"0","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.461149629Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"client":"1","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.461187887Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"client":"2","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.461614863Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"client":"0","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.461638931Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"client":"0","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.46165381Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.461710347Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"client":"1","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.461738606Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"client":"1","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.461753795Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.461767074Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"client":"2","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.461780973Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"client":"2","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"[0]R flush_numbers[0..12]: [122, 123, 118, 122, 122, 122, 115, 122, 125, 113, 124, 124]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.461794973Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"":"downstairs","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"[0]R generation[0..12]: [71, 72, 68, 71, 71, 71, 67, 71, 73, 65, 72, 72]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.461809812Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"":"downstairs","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.461824251Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"":"downstairs","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"[1]R flush_numbers[0..12]: [122, 123, 118, 122, 122, 122, 115, 122, 125, 113, 124, 124]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.4618382Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"":"downstairs","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"[1]R generation[0..12]: [71, 72, 68, 71, 71, 71, 67, 71, 73, 65, 72, 72]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.461852119Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"":"downstairs","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.461866198Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"":"downstairs","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"[2]R flush_numbers[0..12]: [122, 123, 118, 122, 122, 122, 115, 122, 125, 113, 124, 124]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.461883277Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"":"downstairs","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"[2]R generation[0..12]: [71, 72, 68, 71, 71, 71, 67, 71, 73, 65, 72, 72]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.461897167Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"":"downstairs","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.461910966Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"":"downstairs","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.461923715Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"":"downstairs","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.461936794Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"":"downstairs","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"Next flush: 126","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.461949584Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"":"downstairs","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.461962483Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"":"downstairs","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-26T14:30:47.461975722Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.461990431Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"client":"0","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.462004061Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"client":"1","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.46201735Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"client":"2","":"downstairs","session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.462031079Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.462043578Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"74f5eecb-2dff-45ba-a5e9-ca6bae5393ea is now active with session: 7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.462056298Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.462068237Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478,"session_id":"7ac9bd19-4655-41f4-a67c-fbd36e8ddbf7"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:47.462082216Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 19 len 20480 data: 6 4 3 6 8 02/30 Write block 444 len 20480 data: 7 7 8 8 6 03/30 Write block 263 len 28672 data: 8 7 6 6 7 7 6 04/30 Write block 446 len 32768 data: 9 9 7 8 7 6 5 5 05/30 Read block 145 len 32768 06/30 Read block 371 len 36864 07/30 Read block 286 len 12288 08/30 Read block 373 len 24576 09/30 Write block 353 len 28672 data: 9 7 7 5 6 7 6 10/30 Write block 150 len 20480 data: 2 5 5 4 4 11/30 Flush 12/30 Write block 125 len 40960 data: 6 4 5 5 6 6 5 6 6 5 13/30 Write block 400 len 28672 data: 7 8 7 5 5 4 6 14/30 Flush 15/30 Write block 589 len 36864 data: 4 3 2 3 3 5 4 3 4 16/30 Read block 233 len 20480 17/30 Read block 355 len 8192 18/30 Write block 206 len 4096 data: 5 19/30 Read block 66 len 28672 20/30 Write block 357 len 40960 data: 7 8 7 7 5 5 4 5 4 3 21/30 Read block 188 len 8192 22/30 Write block 289 len 20480 data: 9 9 8 9 8 23/30 Read block 51 len 8192 24/30 Flush 25/30 Write block 111 len 24576 data: 10 11 12 10 8 7 26/30 Write block 314 len 12288 data: 10 8 8 27/30 Write block 279 len 8192 data: 12 11 28/30 Read block 317 len 16384 29/30 Read block 64 len 32768 30/30 Write block 282 len 16384 data: 10 8 8 8 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 6 Done Done Done false 26 Acked 1025 Write 3 Done Done Done false 27 Acked 1026 Write 2 Done Done Done false 28 Acked 1027 Read 4 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:30:47.487287539Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1478} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 126 126 126 F F F 1 020-039 74 74 74 126 126 126 F F F 2 040-059 68 68 68 118 118 118 F F F 3 060-079 71 71 71 122 122 122 F F F 4 080-099 71 71 71 122 122 122 F F F 5 100-119 71 71 71 122 122 122 T T T 6 120-139 74 74 74 127 127 127 F F F 7 140-159 74 74 74 126 126 126 F F F 8 160-179 73 73 73 125 125 125 F F F 9 180-199 65 65 65 113 113 113 F F F 10 200-219 74 74 74 128 128 128 F F F 11 220-239 72 72 72 124 124 124 F F F 12 240-259 71 71 71 122 122 122 F F F 13 260-279 74 74 74 126 126 126 T T T 14 280-299 74 74 74 128 128 128 T T T 15 300-319 71 71 71 122 122 122 T T T 16 320-339 68 68 68 119 119 119 F F F 17 340-359 74 74 74 128 128 128 F F F 18 360-379 74 74 74 128 128 128 F F F 19 380-399 72 72 72 124 124 124 F F F 20 400-419 74 74 74 127 127 127 F F F 21 420-439 73 73 73 125 125 125 F F F 22 440-459 74 74 74 126 126 126 F F F 23 460-479 73 73 73 125 125 125 F F F 24 480-499 68 68 68 117 117 117 F F F 25 500-519 61 61 61 106 106 106 F F F 26 520-539 73 73 73 125 125 125 F F F 27 540-559 73 73 73 125 125 125 F F F 28 560-579 72 72 72 124 124 124 F F F 29 580-599 74 74 74 128 128 128 F F F Max gen: 74, Max flush: 128 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 126 126 126 F F F 1 020-039 74 74 74 126 126 126 F F F 2 040-059 68 68 68 118 118 118 F F F 3 060-079 71 71 71 122 122 122 F F F 4 080-099 71 71 71 122 122 122 F F F 5 100-119 75 75 75 129 129 129 F F F 6 120-139 74 74 74 127 127 127 F F F 7 140-159 74 74 74 126 126 126 F F F 8 160-179 73 73 73 125 125 125 F F F 9 180-199 65 65 65 113 113 113 F F F 10 200-219 74 74 74 128 128 128 F F F 11 220-239 72 72 72 124 124 124 F F F 12 240-259 71 71 71 122 122 122 F F F 13 260-279 75 75 75 129 129 129 F F F 14 280-299 75 75 75 129 129 129 F F F 15 300-319 75 75 75 129 129 129 F F F 16 320-339 68 68 68 119 119 119 F F F 17 340-359 74 74 74 128 128 128 F F F 18 360-379 74 74 74 128 128 128 F F F 19 380-399 72 72 72 124 124 124 F F F 20 400-419 74 74 74 127 127 127 F F F 21 420-439 73 73 73 125 125 125 F F F 22 440-459 74 74 74 126 126 126 F F F 23 460-479 73 73 73 125 125 125 F F F 24 480-499 68 68 68 117 117 117 F F F 25 500-519 61 61 61 106 106 106 F F F 26 520-539 73 73 73 125 125 125 F F F 27 540-559 73 73 73 125 125 125 F F F 28 560-579 72 72 72 124 124 124 F F F 29 580-599 74 74 74 128 128 128 F F F Max gen: 75, Max flush: 129 Begin loop 38 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.855344846Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.855574153Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.855600742Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.85563301Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486} {"msg":"Crucible 8caf2479-ee7b-4762-aa7b-f66d4a4083c6 has session id: ed0a00d7-b93a-4512-892e-12e93a4e60e3","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.855862347Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"Upstairs opts: Upstairs UUID: 8caf2479-ee7b-4762-aa7b-f66d4a4083c6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.855910384Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"Crucible stats registered with UUID: 8caf2479-ee7b-4762-aa7b-f66d4a4083c6","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.855935063Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.886748208Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486} {"msg":"8caf2479-ee7b-4762-aa7b-f66d4a4083c6 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.886779426Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.886797385Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"client":"0","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.886824464Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"client":"0","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.886841923Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"client":"1","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.886858771Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"client":"1","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.886875111Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"client":"2","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.886898699Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"client":"2","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.886929778Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.886960906Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"":"io task","client":"0","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.886993884Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"ds_connection connected from Ok(127.0.0.1:47181)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.887110847Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"":"io task","client":"0","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"ds_connection connected from Ok(127.0.0.1:54087)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.887225531Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"ds_connection connected from Ok(127.0.0.1:60459)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.887262639Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.88794526Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"client":"2","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.887977998Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"client":"2","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.887993897Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"client":"0","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.888008297Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"client":"0","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.888022516Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"client":"1","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.888036025Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"client":"1","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.888582074Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"client":"2","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.888703627Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"client":"1","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.88883162Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"client":"0","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.88918133Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"client":"2","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.889209278Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"client":"2","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.889224917Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.889320372Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"client":"1","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.889340391Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"client":"1","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.88935449Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.889441035Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"client":"0","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.889459494Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"client":"0","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"[0]R flush_numbers[0..12]: [126, 126, 118, 122, 122, 129, 127, 126, 125, 113, 128, 124]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.889473634Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"":"downstairs","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"[0]R generation[0..12]: [74, 74, 68, 71, 71, 75, 74, 74, 73, 65, 74, 72]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.889487513Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"":"downstairs","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.889501632Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"":"downstairs","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"[1]R flush_numbers[0..12]: [126, 126, 118, 122, 122, 129, 127, 126, 125, 113, 128, 124]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.889515481Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"":"downstairs","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"[1]R generation[0..12]: [74, 74, 68, 71, 71, 75, 74, 74, 73, 65, 74, 72]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.88952919Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"":"downstairs","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.889543139Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"":"downstairs","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"[2]R flush_numbers[0..12]: [126, 126, 118, 122, 122, 129, 127, 126, 125, 113, 128, 124]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.889556949Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"":"downstairs","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"[2]R generation[0..12]: [74, 74, 68, 71, 71, 75, 74, 74, 73, 65, 74, 72]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.889570808Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"":"downstairs","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.889585107Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"":"downstairs","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.889601786Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"":"downstairs","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.889615386Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"":"downstairs","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"Next flush: 130","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.889628315Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"":"downstairs","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-26T14:30:51.889641064Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"":"downstairs","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.889657043Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.889669172Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"client":"0","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.889682602Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"client":"1","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.889695811Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"client":"2","":"downstairs","session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.88970893Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.88972128Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"8caf2479-ee7b-4762-aa7b-f66d4a4083c6 is now active with session: ed0a00d7-b93a-4512-892e-12e93a4e60e3","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.889734309Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.889746228Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486,"session_id":"ed0a00d7-b93a-4512-892e-12e93a4e60e3"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:51.889758617Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 319 len 36864 data: 8 7 6 5 5 6 6 5 6 02/30 Read block 6 len 28672 03/30 Read block 38 len 16384 04/30 Flush 05/30 Write block 340 len 36864 data: 5 4 5 5 6 6 7 8 8 06/30 Write block 256 len 16384 data: 5 2 3 4 07/30 Read block 9 len 40960 08/30 Flush 09/30 Write block 486 len 28672 data: 6 6 6 6 6 6 7 10/30 Write block 577 len 40960 data: 11 10 7 6 6 7 8 6 6 5 11/30 Write block 171 len 40960 data: 6 5 5 4 4 5 4 4 6 8 12/30 Flush 13/30 Write block 532 len 16384 data: 7 8 7 6 14/30 Write block 518 len 16384 data: 6 6 6 5 15/30 Flush 16/30 Flush 17/30 Write block 595 len 12288 data: 5 4 5 18/30 Flush 19/30 Write block 578 len 8192 data: 11 8 20/30 Write block 137 len 12288 data: 6 4 4 21/30 Read block 41 len 32768 22/30 Read block 485 len 32768 23/30 Write block 305 len 16384 data: 12 10 11 11 24/30 Write block 521 len 20480 data: 6 5 9 9 9 25/30 Write block 474 len 20480 data: 3 4 5 3 5 26/30 Read block 163 len 32768 27/30 Write block 461 len 20480 data: 6 3 5 5 7 28/30 Read block 201 len 36864 29/30 Write block 124 len 8192 data: 6 7 30/30 Read block 119 len 16384 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 19 Acked 1018 Write 2 Done Done Done false 20 Acked 1019 Write 3 Done Done Done false 21 Acked 1020 Read 8 Done Done Done false 22 Acked 1021 Read 8 Done Done Done false 23 Acked 1022 Write 4 Done Done Done false 24 Acked 1023 Write 5 Done Done Done false 25 Acked 1024 Write 5 Done Done Done false 26 Acked 1025 Read 8 Done Done Done false 27 Acked 1026 Write 5 Done Done Done false 28 Acked 1027 Read 9 Done Done Done false 29 Acked 1028 Write 2 Done Done Done false 30 Acked 1029 Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 11 12 11 34 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:30:51.918533098Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1486} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 126 126 126 F F F 1 020-039 74 74 74 126 126 126 F F F 2 040-059 68 68 68 118 118 118 F F F 3 060-079 71 71 71 122 122 122 F F F 4 080-099 71 71 71 122 122 122 F F F 5 100-119 75 75 75 129 129 129 F F F 6 120-139 74 74 74 127 127 127 T T T 7 140-159 74 74 74 126 126 126 F F F 8 160-179 76 76 76 132 132 132 F F F 9 180-199 76 76 76 132 132 132 F F F 10 200-219 74 74 74 128 128 128 F F F 11 220-239 72 72 72 124 124 124 F F F 12 240-259 76 76 76 131 131 131 F F F 13 260-279 75 75 75 129 129 129 F F F 14 280-299 75 75 75 129 129 129 F F F 15 300-319 76 76 76 130 130 130 T T T 16 320-339 76 76 76 130 130 130 F F F 17 340-359 76 76 76 131 131 131 F F F 18 360-379 74 74 74 128 128 128 F F F 19 380-399 72 72 72 124 124 124 F F F 20 400-419 74 74 74 127 127 127 F F F 21 420-439 73 73 73 125 125 125 F F F 22 440-459 74 74 74 126 126 126 F F F 23 460-479 73 73 73 125 125 125 T T T 24 480-499 76 76 76 132 132 132 F F F 25 500-519 76 76 76 133 133 133 F F F 26 520-539 76 76 76 133 133 133 T T T 27 540-559 73 73 73 125 125 125 F F F 28 560-579 76 76 76 132 132 132 T T T 29 580-599 76 76 76 135 135 135 F F F Max gen: 76, Max flush: 135 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 126 126 126 F F F 1 020-039 74 74 74 126 126 126 F F F 2 040-059 68 68 68 118 118 118 F F F 3 060-079 71 71 71 122 122 122 F F F 4 080-099 71 71 71 122 122 122 F F F 5 100-119 75 75 75 129 129 129 F F F 6 120-139 77 77 77 136 136 136 F F F 7 140-159 74 74 74 126 126 126 F F F 8 160-179 76 76 76 132 132 132 F F F 9 180-199 76 76 76 132 132 132 F F F 10 200-219 74 74 74 128 128 128 F F F 11 220-239 72 72 72 124 124 124 F F F 12 240-259 76 76 76 131 131 131 F F F 13 260-279 75 75 75 129 129 129 F F F 14 280-299 75 75 75 129 129 129 F F F 15 300-319 77 77 77 136 136 136 F F F 16 320-339 76 76 76 130 130 130 F F F 17 340-359 76 76 76 131 131 131 F F F 18 360-379 74 74 74 128 128 128 F F F 19 380-399 72 72 72 124 124 124 F F F 20 400-419 74 74 74 127 127 127 F F F 21 420-439 73 73 73 125 125 125 F F F 22 440-459 74 74 74 126 126 126 F F F 23 460-479 77 77 77 136 136 136 F F F 24 480-499 76 76 76 132 132 132 F F F 25 500-519 76 76 76 133 133 133 F F F 26 520-539 77 77 77 136 136 136 F F F 27 540-559 73 73 73 125 125 125 F F F 28 560-579 77 77 77 136 136 136 F F F 29 580-599 76 76 76 135 135 135 F F F Max gen: 77, Max flush: 136 Begin loop 39 Downstairs to restart: 2 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.303478979Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.303841668Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.303870407Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.303895345Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494} {"msg":"Crucible be81ec69-5e2b-43fa-908a-ebfc74f7d678 has session id: d3ace8ff-3788-450d-ab44-e47278688b8d","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.303915974Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"Upstairs opts: Upstairs UUID: be81ec69-5e2b-43fa-908a-ebfc74f7d678, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.303954082Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"Crucible stats registered with UUID: be81ec69-5e2b-43fa-908a-ebfc74f7d678","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.30398075Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.335652657Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494} {"msg":"be81ec69-5e2b-43fa-908a-ebfc74f7d678 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.335684105Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.335702444Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"client":"0","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.335730442Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"client":"0","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.335748291Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"client":"1","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.33576543Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"client":"1","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.335782409Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"client":"2","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.335852855Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"client":"2","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.335888363Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"":"io task","client":"2","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.335915362Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.33593997Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"":"io task","client":"1","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:38701)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.335967299Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"":"io task","client":"1","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:36429)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.335995807Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:41037)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.336021486Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"":"io task","client":"2","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.336859708Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"client":"0","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.336902826Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"client":"0","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.336933274Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"client":"1","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.33700859Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"client":"1","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.337044978Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"client":"2","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.337074726Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"client":"2","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.33752891Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"client":"1","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.337556459Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"client":"2","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.337572998Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"client":"0","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.338091069Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"client":"1","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.338118827Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"client":"1","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.338134646Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.338147565Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"client":"2","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.338161554Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"client":"2","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.338175164Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.338187683Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"client":"0","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.338205092Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"client":"0","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"[0]R flush_numbers[0..12]: [126, 126, 118, 122, 122, 129, 136, 126, 132, 132, 128, 124]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.338220741Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"":"downstairs","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"[0]R generation[0..12]: [74, 74, 68, 71, 71, 75, 77, 74, 76, 76, 74, 72]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.33823459Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"":"downstairs","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.33824882Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"":"downstairs","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"[1]R flush_numbers[0..12]: [126, 126, 118, 122, 122, 129, 136, 126, 132, 132, 128, 124]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.338262659Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"":"downstairs","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"[1]R generation[0..12]: [74, 74, 68, 71, 71, 75, 77, 74, 76, 76, 74, 72]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.338276538Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"":"downstairs","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.338290857Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"":"downstairs","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"[2]R flush_numbers[0..12]: [126, 126, 118, 122, 122, 129, 136, 126, 132, 132, 128, 124]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.338304606Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"":"downstairs","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"[2]R generation[0..12]: [74, 74, 68, 71, 71, 75, 77, 74, 76, 76, 74, 72]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.338318286Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"":"downstairs","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.338332305Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"":"downstairs","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.338345484Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"":"downstairs","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.338358693Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"":"downstairs","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"Next flush: 137","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.338371553Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"":"downstairs","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"All extents match","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2024-09-26T14:30:56.338388042Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"":"downstairs","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.338403801Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.33841591Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"client":"0","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.33842901Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"client":"1","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.338442219Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"client":"2","":"downstairs","session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.338455528Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.338467997Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"be81ec69-5e2b-43fa-908a-ebfc74f7d678 is now active with session: d3ace8ff-3788-450d-ab44-e47278688b8d","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.338480707Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.338492526Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494,"session_id":"d3ace8ff-3788-450d-ab44-e47278688b8d"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:30:56.338504675Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 466 len 32768 data: 8 7 6 7 6 6 4 4 02/30 Flush 03/30 Write block 585 len 16384 data: 7 6 5 4 04/30 Write block 278 len 16384 data: 13 13 12 9 05/30 Read block 242 len 40960 06/30 Read block 438 len 40960 07/30 Read block 159 len 20480 08/30 Read block 441 len 40960 09/30 Read block 48 len 36864 10/30 Flush 11/30 Write block 115 len 36864 data: 9 8 7 7 7 7 7 6 5 12/30 Read block 398 len 32768 13/30 Read block 554 len 40960 14/30 Read block 308 len 28672 15/30 Read block 572 len 12288 16/30 Write block 153 len 32768 data: 5 5 6 11 10 10 10 7 17/30 Write block 362 len 12288 data: 6 5 6 18/30 Read block 323 len 16384 19/30 Read block 305 len 40960 20/30 Write block 571 len 36864 data: 7 6 8 10 10 10 12 12 9 21/30 Read block 176 len 32768 22/30 Read block 274 len 20480 23/30 Flush 24/30 Write block 81 len 28672 data: 9 8 5 5 6 10 9 25/30 Write block 71 len 32768 data: 8 9 10 8 6 8 7 7 26/30 Write block 15 len 28672 data: 7 7 7 7 7 5 4 27/30 Read block 144 len 24576 28/30 Read block 147 len 36864 29/30 Read block 519 len 28672 30/30 Write block 74 len 24576 data: 9 7 9 8 8 10 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 7 Done Done Done false 25 Acked 1024 Write 8 Done Done Done false 26 Acked 1025 Write 7 Done Done Done false 27 Acked 1026 Read 6 Done Done Done false 28 Acked 1027 Read 9 Done Done Done false 29 Acked 1028 Read 7 Done Done Done false 30 Acked 1029 Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:30:56.365929582Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1494} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 126 126 126 T T T 1 020-039 74 74 74 126 126 126 T T T 2 040-059 68 68 68 118 118 118 F F F 3 060-079 71 71 71 122 122 122 T T T 4 080-099 71 71 71 122 122 122 T T T 5 100-119 78 78 78 139 139 139 F F F 6 120-139 78 78 78 139 139 139 F F F 7 140-159 78 78 78 139 139 139 F F F 8 160-179 78 78 78 139 139 139 F F F 9 180-199 76 76 76 132 132 132 F F F 10 200-219 74 74 74 128 128 128 F F F 11 220-239 72 72 72 124 124 124 F F F 12 240-259 76 76 76 131 131 131 F F F 13 260-279 78 78 78 138 138 138 F F F 14 280-299 78 78 78 138 138 138 F F F 15 300-319 77 77 77 136 136 136 F F F 16 320-339 76 76 76 130 130 130 F F F 17 340-359 76 76 76 131 131 131 F F F 18 360-379 78 78 78 139 139 139 F F F 19 380-399 72 72 72 124 124 124 F F F 20 400-419 74 74 74 127 127 127 F F F 21 420-439 73 73 73 125 125 125 F F F 22 440-459 74 74 74 126 126 126 F F F 23 460-479 78 78 78 137 137 137 F F F 24 480-499 76 76 76 132 132 132 F F F 25 500-519 76 76 76 133 133 133 F F F 26 520-539 77 77 77 136 136 136 F F F 27 540-559 73 73 73 125 125 125 F F F 28 560-579 78 78 78 139 139 139 F F F 29 580-599 78 78 78 138 138 138 F F F Max gen: 78, Max flush: 139 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 79 79 79 140 140 140 F F F 1 020-039 79 79 79 140 140 140 F F F 2 040-059 68 68 68 118 118 118 F F F 3 060-079 79 79 79 140 140 140 F F F 4 080-099 79 79 79 140 140 140 F F F 5 100-119 78 78 78 139 139 139 F F F 6 120-139 78 78 78 139 139 139 F F F 7 140-159 78 78 78 139 139 139 F F F 8 160-179 78 78 78 139 139 139 F F F 9 180-199 76 76 76 132 132 132 F F F 10 200-219 74 74 74 128 128 128 F F F 11 220-239 72 72 72 124 124 124 F F F 12 240-259 76 76 76 131 131 131 F F F 13 260-279 78 78 78 138 138 138 F F F 14 280-299 78 78 78 138 138 138 F F F 15 300-319 77 77 77 136 136 136 F F F 16 320-339 76 76 76 130 130 130 F F F 17 340-359 76 76 76 131 131 131 F F F 18 360-379 78 78 78 139 139 139 F F F 19 380-399 72 72 72 124 124 124 F F F 20 400-419 74 74 74 127 127 127 F F F 21 420-439 73 73 73 125 125 125 F F F 22 440-459 74 74 74 126 126 126 F F F 23 460-479 78 78 78 137 137 137 F F F 24 480-499 76 76 76 132 132 132 F F F 25 500-519 76 76 76 133 133 133 F F F 26 520-539 77 77 77 136 136 136 F F F 27 540-559 73 73 73 125 125 125 F F F 28 560-579 78 78 78 139 139 139 F F F 29 580-599 78 78 78 138 138 138 F F F Max gen: 79, Max flush: 140 Begin loop 40 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.731424755Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.7316905Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.731721788Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.731748866Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502} {"msg":"Crucible 0774ec9c-b4b1-4959-9ac6-e571b0b3d918 has session id: 365e3101-3854-4d1d-993a-19dfb36f2676","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.731870719Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"Upstairs opts: Upstairs UUID: 0774ec9c-b4b1-4959-9ac6-e571b0b3d918, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.731922376Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"Crucible stats registered with UUID: 0774ec9c-b4b1-4959-9ac6-e571b0b3d918","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.731942595Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.763023645Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502} {"msg":"0774ec9c-b4b1-4959-9ac6-e571b0b3d918 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.763054523Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.763073112Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"client":"0","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.763100581Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"client":"0","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.763125369Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"client":"1","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.763159107Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"client":"1","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.763191056Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"client":"2","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.763222484Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"client":"2","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.763252132Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"":"io task","client":"2","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.76328213Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"":"io task","client":"0","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.763314518Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"ds_connection connected from Ok(127.0.0.1:47415)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.763347837Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"":"io task","client":"2","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"ds_connection connected from Ok(127.0.0.1:49302)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.763383155Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"ds_connection connected from Ok(127.0.0.1:56412)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.76346325Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"":"io task","client":"0","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.764052627Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"client":"1","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.764079355Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"client":"1","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.764095024Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"client":"2","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.764108904Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"client":"2","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.764122583Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"client":"0","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.764141002Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"client":"0","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.76469076Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"client":"2","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.764856261Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"client":"1","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.7648847Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"client":"0","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.765188332Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"client":"2","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.76522457Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"client":"2","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.765248389Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.765455127Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"client":"0","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.765494545Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"client":"0","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.765522004Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.765550682Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"client":"1","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.76557609Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"client":"1","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"[0]R flush_numbers[0..12]: [140, 140, 118, 140, 140, 139, 139, 139, 139, 132, 128, 124]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.765634647Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"":"downstairs","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"[0]R generation[0..12]: [79, 79, 68, 79, 79, 78, 78, 78, 78, 76, 74, 72]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.765661925Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"":"downstairs","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.765678435Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"":"downstairs","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"[1]R flush_numbers[0..12]: [140, 140, 118, 140, 140, 139, 139, 139, 139, 132, 128, 124]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.765696564Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"":"downstairs","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"[1]R generation[0..12]: [79, 79, 68, 79, 79, 78, 78, 78, 78, 76, 74, 72]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.765710843Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"":"downstairs","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.765724732Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"":"downstairs","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"[2]R flush_numbers[0..12]: [140, 140, 118, 140, 140, 139, 139, 139, 139, 132, 128, 124]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.765738441Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"":"downstairs","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"[2]R generation[0..12]: [79, 79, 68, 79, 79, 78, 78, 78, 78, 76, 74, 72]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.76575306Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"":"downstairs","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.76576718Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"":"downstairs","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.765780499Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"":"downstairs","":"downstairs","session_idWait for a query_work_queue command to finish before sending IO ":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.765797148Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"":"downstairs","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"Next flush: 141","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.765810407Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"":"downstairs","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.765823307Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"":"downstairs","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.765836546Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.765848735Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"client":"0","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.765862184Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"client":"1","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.765875193Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"client":"2","":"downstairs","session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.765888513Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.765902662Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"0774ec9c-b4b1-4959-9ac6-e571b0b3d918 is now active with session: 365e3101-3854-4d1d-993a-19dfb36f2676","v":0,Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "name":"crucible","level":30,"time":"2024-09-26T14:31:00.765921331Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.76593398Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502,"session_id":"365e3101-3854-4d1d-993a-19dfb36f2676"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:00.765946569Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 314 len 28672 data: 11 9 9 9 6 9 8 02/30 Write block 545 len 28672 data: 10 10 8 8 6 7 6 03/30 Write block 395 len 36864 data: 5 7 10 10 9 8 9 8 6 04/30 Read block 206 len 12288 05/30 Read block 36 len 24576 06/30 Read block 133 len 40960 07/30 Read block 239 len 20480 08/30 Read block 263 len 40960 09/30 Write block 286 len 24576 data: 8 7 8 10 10 9 10/30 Flush 11/30 Flush 12/30 Write block 424 len 28672 data: 5 5 6 7 8 7 7 13/30 Read block 299 len 40960 14/30 Write block 163 len 20480 data: 7 7 7 5 6 15/30 Write block 46 len 20480 data: 5 6 8 8 10 16/30 Write block 116 len 28672 data: 9 8 8 8 8 8 7 17/30 Write block 486 len 8192 data: 7 7 18/30 Read block 314 len 16384 19/30 Write block 352 len 36864 data: 8 10 8 8 6 8 9 8 8 20/30 Flush 21/30 Write block 565 len 8192 data: 4 3 22/30 Write block 216 len 20480 data: 9 11 10 10 8 23/30 Write block 340 len 16384 data: 6 5 6 6 24/30 Flush 25/30 Write block 1 len 32768 data: 5 4 6 4 3 3 4 5 26/30 Read block 353 len 28672 27/30 Read block 573 len 4096 28/30 Write block 361 len 8192 data: 6 7 29/30 Read block 161 len 24576 30/30 Write block 206 len 4096 data: 6 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 8 Done Done Done false 26 Acked 1025 Read 7 Done Done Done false 27 Acked 1026 Read 1 Done Done Done false 28 Acked 1027 Write 2 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:31:00.792384811Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1502} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 79 79 79 140 140 140 T T T 1 020-039 79 79 79 140 140 140 F F F 2 040-059 80 80 80 143 143 143 F F F 3 060-079 79 79 79 140 140 140 F F F 4 080-099 79 79 79 140 140 140 F F F 5 100-119 80 80 80 143 143 143 F F F 6 120-139 80 80 80 143 143 143 F F F 7 140-159 78 78 78 139 139 139 F F F 8 160-179 80 80 80 143 143 143 F F F 9 180-199 76 76 76 132 132 132 F F F 10 200-219 80 80 80 144 144 144 T T T 11 220-239 80 80 80 144 144 144 F F F 12 240-259 76 76 76 131 131 131 F F F 13 260-279 78 78 78 138 138 138 F F F 14 280-299 80 80 80 141 141 141 F F F 15 300-319 80 80 80 141 141 141 F F F 16 320-339 80 80 80 141 141 141 F F F 17 340-359 80 80 80 144 144 144 F F F 18 360-379 80 80 80 143 143 143 T T T 19 380-399 80 80 80 141 141 141 F F F 20 400-419 80 80 80 141 141 141 F F F 21 420-439 80 80 80 143 143 143 F F F 22 440-459 74 74 74 126 126 126 F F F 23 460-479 78 78 78 137 137 137 F F F 24 480-499 80 80 80 143 143 143 F F F 25 500-519 76 76 76 133 133 133 F F F 26 520-539 77 77 77 136 136 136 F F F 27 540-559 80 80 80 141 141 141 F F F 28 560-579 80 80 80 144 144 144 F F F 29 580-599 78 78 78 138 138 138 F F F Max gen: 80, Max flush: 144 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 81 81 81 145 145 145 F F F 1 020-039 79 79 79 140 140 140 F F F 2 040-059 80 80 80 143 143 143 F F F 3 060-079 79 79 79 140 140 140 F F F 4 080-099 79 79 79 140 140 140 F F F 5 100-119 80 80 80 143 143 143 F F F 6 120-139 80 80 80 143 143 143 F F F 7 140-159 78 78 78 139 139 139 F F F 8 160-179 80 80 80 143 143 143 F F F 9 180-199 76 76 76 132 132 132 F F F 10 200-219 81 81 81 145 145 145 F F F 11 220-239 80 80 80 144 144 144 F F F 12 240-259 76 76 76 131 131 131 F F F 13 260-279 78 78 78 138 138 138 F F F 14 280-299 80 80 80 141 141 141 F F F 15 300-319 80 80 80 141 141 141 F F F 16 320-339 80 80 80 141 141 141 F F F 17 340-359 80 80 80 144 144 144 F F F 18 360-379 81 81 81 145 145 145 F F F 19 380-399 80 80 80 141 141 141 F F F 20 400-419 80 80 80 141 141 141 F F F 21 420-439 80 80 80 143 143 143 F F F 22 440-459 74 74 74 126 126 126 F F F 23 460-479 78 78 78 137 137 137 F F F 24 480-499 80 80 80 143 143 143 F F F 25 500-519 76 76 76 133 133 133 F F F 26 520-539 77 77 77 136 136 136 F F F 27 540-559 80 80 80 141 141 141 F F F 28 560-579 80 80 80 144 144 144 F F F 29 580-599 78 78 78 138 138 138 F F F Max gen: 81, Max flush: 145 Begin loop 41 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.143572225Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.143926315Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.143966283Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.143997661Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510} {"msg":"Crucible 09517dfa-f885-4f7b-89ec-19f928e37337 has session id: bd911e58-509c-4b9f-b25e-04c0cc3225ee","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.144023559Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"Upstairs opts: Upstairs UUID: 09517dfa-f885-4f7b-89ec-19f928e37337, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.144071977Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"Crucible stats registered with UUID: 09517dfa-f885-4f7b-89ec-19f928e37337","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.144100015Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.17544689Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510} {"msg":"09517dfa-f885-4f7b-89ec-19f928e37337 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.175481098Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.175498557Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"client":"0","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.175522766Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"client":"0","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.175539945Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"client":"1","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.175556614Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"client":"1","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.175573163Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"client":"2","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.175600841Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"client":"2","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.17561909Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.175640679Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.175658378Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:42287)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.175765172Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:35684)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.175815599Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:49378)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.176052696Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.176530198Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"client":"2","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.176565676Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"client":"2","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.176711838Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"client":"1","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.176733287Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"client":"1","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.176747176Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"client":"0","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.176761185Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"client":"0","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.177263487Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"client":"2","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.177308525Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"client":"0","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.177432398Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"client":"1","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.177858663Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"client":"2","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.177887292Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"client":"2","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.177902141Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.178001705Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"client":"0","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.178021234Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"client":"0","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.178034563Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.178106329Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"client":"1","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.178124868Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"client":"1","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"[0]R flush_numbers[0..12]: [145, 140, 143, 140, 140, 143, 143, 139, 143, 132, 145, 144]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.178141237Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"":"downstairs","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"[0]R generation[0..12]: [81, 79, 80, 79, 79, 80, 80, 78, 80, 76, 81, 80]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.178155257Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"":"downstairs","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.178169456Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"":"downstairs","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"[1]R flush_numbers[0..12]: [145, 140, 143, 140, 140, 143, 143, 139, 143, 132, 145, 144]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.178183355Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"":"downstairs","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"[1]R generation[0..12]: [81, 79, 80, 79, 79, 80, 80, 78, 80, 76, 81, 80]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.178196784Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"":"downstairs","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.178210833Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"":"downstairs","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"[2]R flush_numbers[0..12]: [145, 140, 143, 140, 140, 143, 143, 139, 143, 132, 145, 144]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.178224642Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"":"downstairs","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"[2]R generation[0..12]: [81, 79, 80, 79, 79, 80, 80, 78, 80, 76, 81, 80]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.178238132Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"":"downstairs","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.178255631Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"":"downstairs","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.1782689Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"":"downstairs","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.178282379Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"":"downstairs","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"Next flush: 146","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.178295429Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"":"downstairs","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.178308488Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"":"downstairs","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-26T14:31:05.178322027Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.178336736Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"client":"0","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.178350526Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"client":"1","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.178364855Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"client":"2","":"downstairs","session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.178378204Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.178390713Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"09517dfa-f885-4f7b-89ec-19f928e37337 is now active with session: bd911e58-509c-4b9f-b25e-04c0cc3225ee","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.178403683Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.178415692Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510,"session_id":"bd911e58-509c-4b9f-b25e-04c0cc3225ee"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:05.178427971Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 88 len 24576 data: 8 8 9 8 7 7 02/30 Read block 330 len 40960 03/30 Write block 121 len 32768 data: 9 8 6 7 8 5 6 6 04/30 Read block 30 len 36864 05/30 Write block 383 len 40960 data: 7 5 5 4 4 3 3 4 4 4 06/30 Write block 1 len 20480 data: 6 5 7 5 4 07/30 Write block 157 len 4096 data: 11 08/30 Read block 202 len 8192 09/30 Write block 140 len 40960 data: 5 5 3 3 3 4 4 5 2 2 10/30 Read block 593 len 20480 11/30 Read block 173 len 32768 12/30 Read block 123 len 32768 13/30 Read block 30 len 20480 14/30 Write block 567 len 24576 data: 2 4 6 6 8 7 15/30 Write block 202 len 4096 data: 5 16/30 Read block 58 len 4096 17/30 Read block 320 len 4096 18/30 Read block 492 len 16384 19/30 Write block 501 len 16384 data: 9 10 8 7 20/30 Write block 190 len 12288 data: 5 6 7 21/30 Read block 418 len 32768 22/30 Write block 250 len 4096 data: 4 23/30 Write block 304 len 8192 data: 11 13 24/30 Read block 365 len 24576 25/30 Read block 96 len 4096 26/30 Read block 293 len 12288 27/30 Write block 503 len 32768 data: 9 8 7 6 7 6 5 6 28/30 Read block 274 len 4096 29/30 Read block 188 len 32768 30/30 Write block 474 len 28672 data: 4 5 6 4 6 9 9 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Write 6 Done Done Done false 2 Acked 1001 Read 10 Done Done Done false 3 Acked 1002 Write 8 Done Done Done false 4 Acked 1003 Read 9 Done Done Done false 5 Acked 1004 Write 10 Done Done Done false 6 Acked 1005 Write 5 Done Done Done false 7 Acked 1006 Write 1 Done Done Done false 8 Acked 1007 Read 2 Done Done Done false 9 Acked 1008 Write 10 Done Done Done false 10 Acked 1009 Read 5 Done Done Done false 11 Acked 1010 Read 8 Done Done Done false 12 Acked 1011 Read 8 Done Done Done false 13 Acked 1012 Read 5 Done Done Done false 14 Acked 1013 Write 6 Done Done Done false 15 Acked 1014 Write 1 Done Done Done false 16 Acked 1015 Read 1 Done Done Done false 17 Acked 1016 Read 1 Done Done Done false 18 Acked 1017 Read 4 Done Done Done false 19 Acked 1018 Write 4 Done Done Done false 20 Acked 1019 Write 3 Done Done Done false 21 Acked 1020 Read 8 Done Done Done false 22 Acked 1021 Write 1 Done Done Done false 23 Acked 1022 Write 2 Done Done Done false 24 Acked 1023 Read 6 Done Done Done false 25 Acked 1024 Read 1 Done Done Done false 26 Acked 1025 Read 3 Done Done Done false 27 Acked 1026 Write 8 Done Done Done false 28 Acked 1027 Read 1 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:31:05.197483892Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1510} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 81 81 81 145 145 145 T T T 1 020-039 79 79 79 140 140 140 F F F 2 040-059 80 80 80 143 143 143 F F F 3 060-079 79 79 79 140 140 140 F F F 4 080-099 79 79 79 140 140 140 T T T 5 100-119 80 80 80 143 143 143 F F F 6 120-139 80 80 80 143 143 143 T T T 7 140-159 78 78 78 139 139 139 T T T 8 160-179 80 80 80 143 143 143 F F F 9 180-199 76 76 76 132 132 132 T T T 10 200-219 81 81 81 145 145 145 T T T 11 220-239 80 80 80 144 144 144 F F F 12 240-259 76 76 76 131 131 131 T T T 13 260-279 78 78 78 138 138 138 F F F 14 280-299 80 80 80 141 141 141 F F F 15 300-319 80 80 80 141 141 141 T T T 16 320-339 80 80 80 141 141 141 F F F 17 340-359 80 80 80 144 144 144 F F F 18 360-379 81 81 81 145 145 145 F F F 19 380-399 80 80 80 141 141 141 T T T 20 400-419 80 80 80 141 141 141 F F F 21 420-439 80 80 80 143 143 143 F F F 22 440-459 74 74 74 126 126 126 F F F 23 460-479 78 78 78 137 137 137 T T T 24 480-499 80 80 80 143 143 143 T T T 25 500-519 76 76 76 133 133 133 T T T 26 520-539 77 77 77 136 136 136 F F F 27 540-559 80 80 80 141 141 141 F F F 28 560-579 80 80 80 144 144 144 T T T 29 580-599 78 78 78 138 138 138 F F F Max gen: 81, Max flush: 145 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 146 146 146 F F F 1 020-039 79 79 79 140 140 140 F F F 2 040-059 80 80 80 143 143 143 F F F 3 060-079 79 79 79 140 140 140 F F F 4 080-099 82 82 82 146 146 146 F F F 5 100-119 80 80 80 143 143 143 F F F 6 120-139 82 82 82 146 146 146 F F F 7 140-159 82 82 82 146 146 146 F F F 8 160-179 80 80 80 143 143 143 F F F 9 180-199 82 82 82 146 146 146 F F F 10 200-219 82 82 82 146 146 146 F F F 11 220-239 80 80 80 144 144 144 F F F 12 240-259 82 82 82 146 146 146 F F F 13 260-279 78 78 78 138 138 138 F F F 14 280-299 80 80 80 141 141 141 F F F 15 300-319 82 82 82 146 146 146 F F F 16 320-339 80 80 80 141 141 141 F F F 17 340-359 80 80 80 144 144 144 F F F 18 360-379 81 81 81 145 145 145 F F F 19 380-399 82 82 82 146 146 146 F F F 20 400-419 80 80 80 141 141 141 F F F 21 420-439 80 80 80 143 143 143 F F F 22 440-459 74 74 74 126 126 126 F F F 23 460-479 82 82 82 146 146 146 F F F 24 480-499 82 82 82 146 146 146 F F F 25 500-519 82 82 82 146 146 146 F F F 26 520-539 77 77 77 136 136 136 F F F 27 540-559 80 80 80 141 141 141 F F F 28 560-579 82 82 82 146 146 146 F F F 29 580-599 78 78 78 138 138 138 F F F Max gen: 82, Max flush: 146 Begin loop 42 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.681202741Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.681505104Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.681529052Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.681555571Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518} {"msg":"Crucible d8d31c85-b401-4b32-9d3e-dd3898d10c34 has session id: e84f159d-99fc-4313-bb41-f0a7f45df2c7","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.681720171Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"Upstairs opts: Upstairs UUID: d8d31c85-b401-4b32-9d3e-dd3898d10c34, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.681770409Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"Crucible stats registered with UUID: d8d31c85-b401-4b32-9d3e-dd3898d10c34","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.681796447Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.713217418Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518} {"msg":"d8d31c85-b401-4b32-9d3e-dd3898d10c34 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.713246026Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.713264095Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"client":"0","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.713291174Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"client":"0","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.713308582Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"client":"1","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.713367909Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"client":"1","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.713442915Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"client":"2","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.713461904Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"client":"2","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.713479893Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.713494872Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.713508541Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"ds_connection connected from Ok(127.0.0.1:39990)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.71352264Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"ds_connection connected from Ok(127.0.0.1:45531)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.713693811Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"ds_connection connected from Ok(127.0.0.1:57908)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.713771166Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.714344014Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"client":"0","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.714371142Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"client":"0","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.714386252Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"client":"1","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.714400841Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"client":"1","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.71441444Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"client":"2","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.714431929Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"client":"2","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.714976798Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"client":"2","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.715135559Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"client":"1","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.715266902Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"client":"0","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.715660039Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"client":"1","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.715698187Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"client":"1","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.715725166Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.715749354Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"client":"2","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.715775933Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"client":"2","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.715798981Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.71581882Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"client":"0","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.715842089Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"client":"0","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"[0]R flush_numbers[0..12]: [146, 140, 143, 140, 146, 143, 146, 146, 143, 146, 146, 144]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.715867188Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"":"downstairs","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"[0]R generation[0..12]: [82, 79, 80, 79, 82, 80, 82, 82, 80, 82, 82, 80]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.715891626Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"":"downstairs","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.715914785Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"":"downstairs","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"[1]R flush_numbers[0..12]: [146, 140, 143, 140, 146, 143, 146, 146, 143, 146, 146, 144]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.715937674Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"":"downstairs","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"[1]R generation[0..12]: [82, 79, 80, 79, 82, 80, 82, 82, 80, 82, 82, 80]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.715962102Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"":"downstairs","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.715989301Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"":"downstairs","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"[2]R flush_numbers[0..12]: [146, 140, 143, 140, 146, 143, 146, 146, 143, 146, 146, 144]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.716019219Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"":"downstairs","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"[2]R generation[0..12]: [82, 79, 80, 79, 82, 80, 82, 82, 80, 82, 82, 80]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.716044067Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"":"downstairs","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.716071326Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"":"downstairs","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"Max found gen is 83","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.716102274Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"":"downstairs","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"Generation requested: 84 >= found:83","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.716129423Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"":"downstairs","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"Next flush: 147","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.716154691Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"":"downstairs","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.71618099Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"":"downstairs","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.716208728Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.716233737Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"client":"0","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "time":"2024-09-26T14:31:09.716260495Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"client":"1","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.716293304Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"client":"2","":"downstairs","session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.716318212Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.716331871Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"d8d31c85-b401-4b32-9d3e-dd3898d10c34 is now active with session: e84f159d-99fc-4313-bb41-f0a7f45df2c7","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.716344981Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.71635696Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518,"session_id":"e84f159d-99fc-4313-bb41-f0a7f45df2c7"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:09.716369199Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 4 len 8192 data: 6 5 02/30 Write block 568 len 40960 data: 5 7 7 9 8 9 11 11 11 13 03/30 Write block 155 len 4096 data: 7 04/30 Write block 519 len 8192 data: 7 7 05/30 Read block 565 len 4096 06/30 Write block 209 len 8192 data: 5 5 07/30 Read block 407 len 28672 08/30 Read block 429 len 12288 09/30 Read block 446 len 16384 10/30 Write block 86 len 36864 data: 11 10 9 9 10 9 8 8 3 11/30 Write block 244 len 24576 data: 6 7 6 8 6 6 12/30 Read block 37 len 24576 13/30 Flush 14/30 Flush 15/30 Write block 508 len 24576 data: 7 6 7 6 6 4 16/30 Read block 395 len 4096 17/30 Write block 537 len 28672 data: 6 7 7 9 7 6 7 18/30 Write block 413 len 24576 data: 5 6 6 7 6 5 19/30 Read block 125 len 28672 20/30 Read block 56 len 40960 21/30 Write block 416 len 40960 data: 8 7 6 5 5 6 6 7 6 6 22/30 Read block 312 len 36864 23/30 Read block 75 len 8192 24/30 Read block 114 len 36864 25/30 Flush 26/30 Write block 432 len 8192 data: 7 8 27/30 Read block 120 len 28672 28/30 Write block 576 len 12288 data: 12 14 13 29/30 Read block 198 len 20480 30/30 Read block 367 len 36864 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 2 Done Done Done false 27 Acked 1026 Read 7 Done Done Done false 28 Acked 1027 Write 3 Done Done Done false 29 Acked 1028 Read 5 Done Done Done false 30 Acked 1029 Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 4 5 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:31:09.740617046Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1518} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 147 147 147 F F F 1 020-039 79 79 79 140 140 140 F F F 2 040-059 80 80 80 143 143 143 F F F 3 060-079 79 79 79 140 140 140 F F F 4 080-099 84 84 84 147 147 147 F F F 5 100-119 80 80 80 143 143 143 F F F 6 120-139 82 82 82 146 146 146 F F F 7 140-159 84 84 84 147 147 147 F F F 8 160-179 80 80 80 143 143 143 F F F 9 180-199 82 82 82 146 146 146 F F F 10 200-219 84 84 84 147 147 147 F F F 11 220-239 80 80 80 144 144 144 F F F 12 240-259 84 84 84 147 147 147 F F F 13 260-279 78 78 78 138 138 138 F F F 14 280-299 80 80 80 141 141 141 F F F 15 300-319 82 82 82 146 146 146 F F F 16 320-339 80 80 80 141 141 141 F F F 17 340-359 80 80 80 144 144 144 F F F 18 360-379 81 81 81 145 145 145 F F F 19 380-399 82 82 82 146 146 146 F F F 20 400-419 84 84 84 149 149 149 F F F 21 420-439 84 84 84 149 149 149 T T T 22 440-459 74 74 74 126 126 126 F F F 23 460-479 82 82 82 146 146 146 F F F 24 480-499 82 82 82 146 146 146 F F F 25 500-519 84 84 84 149 149 149 F F F 26 520-539 84 84 84 149 149 149 F F F 27 540-559 84 84 84 149 149 149 F F F 28 560-579 84 84 84 147 147 147 T T T 29 580-599 78 78 78 138 138 138 F F F Max gen: 84, Max flush: 149 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 147 147 147 F F F 1 020-039 79 79 79 140 140 140 F F F 2 040-059 80 80 80 143 143 143 F F F 3 060-079 79 79 79 140 140 140 F F F 4 080-099 84 84 84 147 147 147 F F F 5 100-119 80 80 80 143 143 143 F F F 6 120-139 82 82 82 146 146 146 F F F 7 140-159 84 84 84 147 147 147 F F F 8 160-179 80 80 80 143 143 143 F F F 9 180-199 82 82 82 146 146 146 F F F 10 200-219 84 84 84 147 147 147 F F F 11 220-239 80 80 80 144 144 144 F F F 12 240-259 84 84 84 147 147 147 F F F 13 260-279 78 78 78 138 138 138 F F F 14 280-299 80 80 80 141 141 141 F F F 15 300-319 82 82 82 146 146 146 F F F 16 320-339 80 80 80 141 141 141 F F F 17 340-359 80 80 80 144 144 144 F F F 18 360-379 81 81 81 145 145 145 F F F 19 380-399 82 82 82 146 146 146 F F F 20 400-419 84 84 84 149 149 149 F F F 21 420-439 85 85 85 150 150 150 F F F 22 440-459 74 74 74 126 126 126 F F F 23 460-479 82 82 82 146 146 146 F F F 24 480-499 82 82 82 146 146 146 F F F 25 500-519 84 84 84 149 149 149 F F F 26 520-539 84 84 84 149 149 149 F F F 27 540-559 84 84 84 149 149 149 F F F 28 560-579 85 85 85 150 150 150 F F F 29 580-599 78 78 78 138 138 138 F F F Max gen: 85, Max flush: 150 Begin loop 43 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.115962191Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.116276573Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.116310931Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.116342489Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526} {"msg":"Crucible 02f0b007-8c84-4c4a-aa40-3d1cf29353d8 has session id: af807b1a-98b6-4b09-b26a-8ad97459feb5","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.116410625Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"Upstairs opts: Upstairs UUID: 02f0b007-8c84-4c4a-aa40-3d1cf29353d8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.116468802Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"Crucible stats registered with UUID: 02f0b007-8c84-4c4a-aa40-3d1cf29353d8","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.11649807Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.147891012Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526} {"msg":"02f0b007-8c84-4c4a-aa40-3d1cf29353d8 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.147922941Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.1479425Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"client":"0","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.147986677Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"client":"0","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.148013576Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"client":"1","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.148086891Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"client":"1","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.148180746Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"client":"2","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.148244512Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"client":"2","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.14829338Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.148343877Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.148392904Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"ds_connection connected from Ok(127.0.0.1:63570)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.148438441Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"ds_connection connected from Ok(127.0.0.1:59175)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.148484079Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"ds_connection connected from Ok(127.0.0.1:38264)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.148573634Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.149268034Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"client":"2","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.149296383Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"client":"2","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.149312232Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"client":"1","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.149326361Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"client":"1","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.14933981Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"client":"0","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.14935344Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"client":"0","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.149902238Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"client":"2","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.150148435Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"client":"1","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.150177533Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"client":"0","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.150497485Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"client":"2","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.150534503Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"client":"2","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.150559271Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.150666715Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"client":"1","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.150692784Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"client":"1","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.150710983Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.150733051Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"client":"0","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.150747051Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"client":"0","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"[0]R flush_numbers[0..12]: [147, 140, 143, 140, 147, 143, 146, 147, 143, 146, 147, 144]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.150803238Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"":"downstairs","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"[0]R generation[0..12]: [84, 79, 80, 79, 84, 80, 82, 84, 80, 82, 84, 80]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.150823626Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"":"downstairs","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.150838156Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"":"downstairs","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"[1]R flush_numbers[0..12]: [147, 140, 143, 140, 147, 143, 146, 147, 143, 146, 147, 144]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.150851995Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"":"downstairs","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"[1]R generation[0..12]: [84, 79, 80, 79, 84, 80, 82, 84, 80, 82, 84, 80]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.150865434Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"":"downstairs","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.150879233Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"":"downstairs","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"[2]R flush_numbers[0..12]: [147, 140, 143, 140, 147, 143, 146, 147, 143, 146, 147, 144]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.150893232Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"":"downstairs","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"[2]R generation[0..12]: [84, 79, 80, 79, 84, 80, 82, 84, 80, 82, 84, 80]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.150906672Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"":"downstairs","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.150920711Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"":"downstairs","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.15093349Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"":"downstairs","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.150947599Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"":"downstairs","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 151","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.150968738Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"":"downstairs","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.150982587Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"":"downstairs","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.150995827Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.151007886Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"client":"0","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.151021175Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"client":"1","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.151034144Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"client":"2","":"downstairs","session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.151047284Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.151059463Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"02f0b007-8c84-4c4a-aa40-3d1cf29353d8 is now active with session: af807b1a-98b6-4b09-b26a-8ad97459feb5","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.151072142Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.151084022Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526,"session_id":"af807b1a-98b6-4b09-b26a-8ad97459feb5"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:14.151096101Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 391 len 12288 data: 5 5 4 02/30 Write block 270 len 28672 data: 5 4 4 4 7 10 11 03/30 Write block 547 len 24576 data: 9 9 7 8 7 5 04/30 Read block 449 len 16384 05/30 Read block 329 len 8192 06/30 Write block 559 len 20480 data: 4 5 4 4 4 07/30 Write block 536 len 32768 data: 6 7 8 8 10 8 7 8 08/30 Flush 09/30 Write block 363 len 36864 data: 6 7 5 4 4 3 5 5 5 10/30 Read block 268 len 16384 11/30 Write block 228 len 4096 data: 6 12/30 Read block 218 len 12288 13/30 Write block 401 len 12288 data: 10 9 7 14/30 Read block 27 len 32768 15/30 Write block 316 len 8192 data: 10 10 16/30 Write block 420 len 4096 data: 6 17/30 Read block 484 len 20480 18/30 Write block 397 len 36864 data: 11 11 10 9 11 10 8 6 5 19/30 Read block 234 len 4096 20/30 Read block 327 len 20480 21/30 Read block 292 len 16384 22/30 Write block 112 len 24576 data: 12 13 11 10 10 9 23/30 Read block 123 len 24576 24/30 Read block 524 len 32768 25/30 Write block 512 len 40960 data: 7 5 5 4 3 5 7 8 8 7 26/30 Write block 155 len 8192 data: 8 12 27/30 Write block 584 len 4096 data: 7 28/30 Read block 21 len 12288 29/30 Read block 408 len 20480 30/30 Write block 144 len 16384 data: 4 5 5 6 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 9 Acked 1008 Write 9 Done Done Done false 10 Acked 1009 Read 4 Done Done Done false 11 Acked 1010 Write 1 Done Done Done false 12 Acked 1011 Read 3 Done Done Done false 13 Acked 1012 Write 3 Done Done Done false 14 Acked 1013 Read 8 Done Done Done false 15 Acked 1014 Write 2 Done Done Done false 16 Acked 1015 Write 1 Done Done Done false 17 Acked 1016 Read 5 Done Done Done false 18 Acked 1017 Write 9 Done Done Done false 19 Acked 1018 Read 1 Done Done Done false 20 Acked 1019 Read 5 Done Done Done false 21 Acked 1020 Read 4 Done Done Done false 22 Acked 1021 Write 6 Done Done Done false 23 Acked 1022 Read 6 Done Done Done false 24 Acked 1023 Read 8 Done Done Done false 25 Acked 1024 Write 10 Done Done Done false 26 Acked 1025 Write 2 Done Done Done false 27 Acked 1026 Write 1 Done Done Done false 28 Acked 1027 Read 3 Done Sent Done false 29 Acked 1028 Read 5 Done Sent Sent false 30 Acked 1029 Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 3 2 6 Done 21 19 20 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1007 1007 1007 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:31:14.173676332Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1526} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 147 147 147 F F F 1 020-039 79 79 79 140 140 140 F F F 2 040-059 80 80 80 143 143 143 F F F 3 060-079 79 79 79 140 140 140 F F F 4 080-099 84 84 84 147 147 147 F F F 5 100-119 80 80 80 143 143 143 T T T 6 120-139 82 82 82 146 146 146 F F F 7 140-159 84 84 84 147 147 147 T T T 8 160-179 80 80 80 143 143 143 F F F 9 180-199 82 82 82 146 146 146 F F F 10 200-219 84 84 84 147 147 147 F F F 11 220-239 80 80 80 144 144 144 T T T 12 240-259 84 84 84 147 147 147 F F F 13 260-279 86 86 86 151 151 151 F F F 14 280-299 80 80 80 141 141 141 F F F 15 300-319 82 82 82 146 146 146 T T T 16 320-339 80 80 80 141 141 141 F F F 17 340-359 80 80 80 144 144 144 F F F 18 360-379 81 81 81 145 145 145 T T T 19 380-399 86 86 86 151 151 151 T T T 20 400-419 84 84 84 149 149 149 T T T 21 420-439 85 85 85 150 150 150 T T T 22 440-459 74 74 74 126 126 126 F F F 23 460-479 82 82 82 146 146 146 F F F 24 480-499 82 82 82 146 146 146 F F F 25 500-519 84 84 84 149 149 149 T T T 26 520-539 86 86 86 151 151 151 T T T 27 540-559 86 86 86 151 151 151 F F F 28 560-579 86 86 86 151 151 151 F F F 29 580-599 78 78 78 138 138 138 T T T Max gen: 86, Max flush: 151 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 147 147 147 F F F 1 020-039 79 79 79 140 140 140 F F F 2 040-059 80 80 80 143 143 143 F F F 3 060-079 79 79 79 140 140 140 F F F 4 080-099 84 84 84 147 147 147 F F F 5 100-119 87 87 87 152 152 152 F F F 6 120-139 82 82 82 146 146 146 F F F 7 140-159 87 87 87 152 152 152 F F F 8 160-179 80 80 80 143 143 143 F F F 9 180-199 82 82 82 146 146 146 F F F 10 200-219 84 84 84 147 147 147 F F F 11 220-239 87 87 87 152 152 152 F F F 12 240-259 84 84 84 147 147 147 F F F 13 260-279 86 86 86 151 151 151 F F F 14 280-299 80 80 80 141 141 141 F F F 15 300-319 87 87 87 152 152 152 F F F 16 320-339 80 80 80 141 141 141 F F F 17 340-359 80 80 80 144 144 144 F F F 18 360-379 87 87 87 152 152 152 F F F 19 380-399 87 87 87 152 152 152 F F F 20 400-419 87 87 87 152 152 152 F F F 21 420-439 87 87 87 152 152 152 F F F 22 440-459 74 74 74 126 126 126 F F F 23 460-479 82 82 82 146 146 146 F F F 24 480-499 82 82 82 146 146 146 F F F 25 500-519 87 87 87 152 152 152 F F F 26 520-539 87 87 87 152 152 152 F F F 27 540-559 86 86 86 151 151 151 F F F 28 560-579 86 86 86 151 151 151 F F F 29 580-599 87 87 87 152 152 152 F F F Max gen: 87, Max flush: 152 Begin loop 44 Downstairs to restart: 2 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.641549316Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.641928214Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.641955293Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.641979961Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534} {"msg":"Crucible d5597d11-e79f-4d57-be33-3de3b85a0f4d has session id: 2bc131be-87a5-423c-ad17-55f6fb5525be","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.642042718Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"Upstairs opts: Upstairs UUID: d5597d11-e79f-4d57-be33-3de3b85a0f4d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.642209218Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"Crucible stats registered with UUID: d5597d11-e79f-4d57-be33-3de3b85a0f4d","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.642250136Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.673590311Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534} {"msg":"d5597d11-e79f-4d57-be33-3de3b85a0f4d active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.673625599Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.673643578Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"client":"0","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.673682766Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"client":"0","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.673715774Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"client":"1","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.673748712Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"client":"1","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.67378201Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"client":"2","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.673821928Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"client":"2","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.673854306Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"":"io task","client":"2","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.673887854Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.673919913Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"":"io task","client":"1","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"ds_connection connected from Ok(127.0.0.1:42680)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.674114322Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"":"io task","client":"2","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"ds_connection connected from Ok(127.0.0.1:64587)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.674162429Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"ds_connection connected from Ok(127.0.0.1:42281)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.674201677Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"":"io task","client":"1","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.674542837Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"client":"0","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.674589735Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"client":"0","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.674774414Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"client":"1","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.674803562Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"client":"1","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.674818582Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"client":"2","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.674833011Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"client":"2","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.675314363Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"client":"0","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.675344942Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"client":"1","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.675646015Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"client":"2","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.675924669Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"client":"1","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.675953937Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"client":"1","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.675969976Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.676036142Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"client":"0","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.676055901Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"client":"0","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.676069651Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.676144576Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"client":"2","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.676163505Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"client":"2","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"[0]R flush_numbers[0..12]: [147, 140, 143, 140, 147, 152, 146, 152, 143, 146, 147, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.676177565Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"":"downstairs","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"[0]R generation[0..12]: [84, 79, 80, 79, 84, 87, 82, 87, 80, 82, 84, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.676191394Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"":"downstairs","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.676205573Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"":"downstairs","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"[1]R flush_numbers[0..12]: [147, 140, 143, 140, 147, 152, 146, 152, 143, 146, 147, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.676219312Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"":"downstairs","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"[1]R generation[0..12]: [84, 79, 80, 79, 84, 87, 82, 87, 80, 82, 84, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.676232671Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"":"downstairs","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.676246691Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"":"downstairs","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"[2]R flush_numbers[0..12]: [147, 140, 143, 140, 147, 152, 146, 152, 143, 146, 147, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.67626061Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"":"downstairs","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"[2]R generation[0..12]: [84, 79, 80, 79, 84, 87, 82, 87, 80, 82, 84, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.676274279Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"":"downstairs","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.676288058Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"":"downstairs","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.676301018Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"":"downstairs","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.676314197Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"":"downstairs","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"Next flush: 153","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.676327136Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"":"downstairs","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.676343095Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"":"downstairs","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.676356744Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"Wait for a query_work_queue command to finish before sending IO } {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.676372323Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"client":"0","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.676385513Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"client":"1","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.676399632Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"client":"2","":"downstairs","session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.676412881Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.676425111Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"d5597d11-e79f-4d57-be33-3de3b85a0f4d is now active with session: 2bc131be-87a5-423c-ad17-55f6fb5525be","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.67643793Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.676450049Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534,"session_id":"2bc131be-87a5-423c-ad17-55f6fb5525be"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:18.676462458Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 472 len 24576 data: 5 5 5 6 7 5 02/30 Read block 309 len 8192 03/30 Read block 58 len 4096 04/30 Write block 531 len 36864 data: 7 8 9 8 7 7 8 9 9 05/30 Write block 303 len 16384 data: 12 12 14 11 06/30 Read block 36 len 16384 07/30 Write block 327 len 40960 data: 7 8 7 6 7 5 6 6 4 4 08/30 Read block 217 len 8192 09/30 Read block 185 len 28672 10/30 Write block 204 len 8192 data: 6 6 11/30 Read block 165 len 36864 12/30 Read block 412 len 36864 13/30 Read block 533 len 24576 14/30 Write block 420 len 32768 data: 7 7 7 8 7 7 7 8 15/30 Read block 290 len 28672 16/30 Write block 269 len 36864 data: 7 6 5 5 5 8 11 12 12 17/30 Read block 513 len 20480 18/30 Read block 225 len 12288 19/30 Write block 420 len 24576 data: 8 8 8 9 8 8 20/30 Flush 21/30 Write block 105 len 20480 data: 7 6 7 9 8 22/30 Write block 131 len 24576 data: 6 7 7 6 5 6 23/30 Write block 89 len 32768 data: 10 11 10 9 9 4 2 2 24/30 Flush 25/30 Write block 544 len 32768 data: 9 11 11 10 10 8 9 8 26/30 Read block 115 len 24576 27/30 Read block 208 len 24576 28/30 Write block 494 len 8192 data: 9 9 29/30 Read block 63 len 32768 30/30 Write block 284 len 32768 data: 9 9 9 8 9 11 11 10 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 8 Done Done Done false 26 Acked 1025 Read 6 Done Done Done false 27 Acked 1026 Read 6 Done Done Done false 28 Acked 1027 Write 2 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:31:18.701490191Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1534} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 147 147 147 F F F 1 020-039 79 79 79 140 140 140 F F F 2 040-059 80 80 80 143 143 143 F F F 3 060-079 79 79 79 140 140 140 F F F 4 080-099 88 88 88 154 154 154 F F F 5 100-119 88 88 88 154 154 154 F F F 6 120-139 88 88 88 154 154 154 F F F 7 140-159 87 87 87 152 152 152 F F F 8 160-179 80 80 80 143 143 143 F F F 9 180-199 82 82 82 146 146 146 F F F 10 200-219 88 88 88 153 153 153 F F F 11 220-239 87 87 87 152 152 152 F F F 12 240-259 84 84 84 147 147 147 F F F 13 260-279 88 88 88 153 153 153 F F F 14 280-299 80 80 80 141 141 141 T T T 15 300-319 88 88 88 153 153 153 F F F 16 320-339 88 88 88 153 153 153 F F F 17 340-359 80 80 80 144 144 144 F F F 18 360-379 87 87 87 152 152 152 F F F 19 380-399 87 87 87 152 152 152 F F F 20 400-419 87 87 87 152 152 152 F F F 21 420-439 88 88 88 153 153 153 F F F 22 440-459 74 74 74 126 126 126 F F F 23 460-479 88 88 88 153 153 153 F F F 24 480-499 82 82 82 146 146 146 T T T 25 500-519 87 87 87 152 152 152 F F F 26 520-539 88 88 88 153 153 153 F F F 27 540-559 86 86 86 151 151 151 T T T 28 560-579 86 86 86 151 151 151 F F F 29 580-599 87 87 87 152 152 152 F F F Max gen: 88, Max flush: 154 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 147 147 147 F F F 1 020-039 79 79 79 140 140 140 F F F 2 040-059 80 80 80 143 143 143 F F F 3 060-079 79 79 79 140 140 140 F F F 4 080-099 88 88 88 154 154 154 F F F 5 100-119 88 88 88 154 154 154 F F F 6 120-139 88 88 88 154 154 154 F F F 7 140-159 87 87 87 152 152 152 F F F 8 160-179 80 80 80 143 143 143 F F F 9 180-199 82 82 82 146 146 146 F F F 10 200-219 88 88 88 153 153 153 F F F 11 220-239 87 87 87 152 152 152 F F F 12 240-259 84 84 84 147 147 147 F F F 13 260-279 88 88 88 153 153 153 F F F 14 280-299 89 89 89 155 155 155 F F F 15 300-319 88 88 88 153 153 153 F F F 16 320-339 88 88 88 153 153 153 F F F 17 340-359 80 80 80 144 144 144 F F F 18 360-379 87 87 87 152 152 152 F F F 19 380-399 87 87 87 152 152 152 F F F 20 400-419 87 87 87 152 152 152 F F F 21 420-439 88 88 88 153 153 153 F F F 22 440-459 74 74 74 126 126 126 F F F 23 460-479 88 88 88 153 153 153 F F F 24 480-499 89 89 89 155 155 155 F F F 25 500-519 87 87 87 152 152 152 F F F 26 520-539 88 88 88 153 153 153 F F F 27 540-559 89 89 89 155 155 155 F F F 28 560-579 86 86 86 151 151 151 F F F 29 580-599 87 87 87 152 152 152 F F F Max gen: 89, Max flush: 155 Begin loop 45 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.062023734Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.062344036Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.062377624Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.062410992Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542} {"msg":"Crucible 4301f2b9-0631-4f66-b5e4-38684bcc94e6 has session id: a3eb23e5-a154-45fd-84ec-88e7f55b4a6e","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.062528416Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"Upstairs opts: Upstairs UUID: 4301f2b9-0631-4f66-b5e4-38684bcc94e6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.062578253Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"Crucible stats registered with UUID: 4301f2b9-0631-4f66-b5e4-38684bcc94e6","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.062602291Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.093846622Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542} {"msg":"4301f2b9-0631-4f66-b5e4-38684bcc94e6 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.09388266Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.093911758Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"client":"0","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.093936247Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"client":"0","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.093954296Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"client":"1","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.093971455Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"client":"1","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.093988044Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"client":"2","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.094004883Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"client":"2","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.094021812Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.094039031Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.0940559Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"":"io task","client":"1","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"ds_connection connected from Ok(127.0.0.1:39462)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.094125856Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"ds_connection connected from Ok(127.0.0.1:63114)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.094169214Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"":"io task","client":"1","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"ds_connection connected from Ok(127.0.0.1:46787)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.094203702Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.095084462Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"client":"1","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.095134189Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"client":"1","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.095172197Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"client":"2","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.095245593Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"client":"2","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.095280951Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"client":"0","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.095311259Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"client":"0","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.09565629Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"client":"2","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.095689908Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"client":"1","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.095895586Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"client":"0","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.096265025Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"client":"1","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.096295433Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"client":"1","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.096322152Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.096339181Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"client":"2","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.09635304Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"client":"2","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.096366529Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.096383038Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"client":"0","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.096396968Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"client":"0","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"[0]R flush_numbers[0..12]: [147, 140, 143, 140, 154, 154, 154, 152, 143, 146, 153, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.096438415Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"":"downstairs","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"[0]R generation[0..12]: [84, 79, 80, 79, 88, 88, 88, 87, 80, 82, 88, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.096455484Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"":"downstairs","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.096473043Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"":"downstairs","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"[1]R flush_numbers[0..12]: [147, 140, 143, 140, 154, 154, 154, 152, 143, 146, 153, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.096487343Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"":"downstairs","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"[1]R generation[0..12]: [84, 79, 80, 79, 88, 88, 88, 87, 80, 82, 88, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.096501002Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"":"downstairs","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.096514971Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"":"downstairs","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"[2]R flush_numbers[0..12]: [147, 140, 143, 140, 154, 154, 154, 152, 143, 146, 153, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.09652871Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"":"downstairs","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"[2]R generation[0..12]: [84, 79, 80, 79, 88, 88, 88, 87, 80, 82, 88, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.096542289Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"":"downstairs","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.096556249Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"":"downstairs","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.096569198Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"":"downstairs","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.096582887Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"":"downstairs","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"Next flush: 156","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.096595946Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"":"downstairs","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"All extents match","v":0,"nameWait for a query_work_queue command to finish before sending IO ":"crucible","level":30,"time":"2024-09-26T14:31:23.096612505Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"":"downstairs","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.096625895Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.096637854Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"client":"0","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.096651303Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"client":"1","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.096666452Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"client":"2","":"downstairs","session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.096679912Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.096692351Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"4301f2b9-0631-4f66-b5e4-38684bcc94e6 is now active with session: a3eb23e5-a154-45fd-84ec-88e7f55b4a6e","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.09670529Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.096717199Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542,"session_id":"a3eb23e5-a154-45fd-84ec-88e7f55b4a6e"} {"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:23.096734329Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 47 len 4096 data: 7 02/30 Write block 520 len 4096 data: 9 03/30 Write block 91 len 16384 data: 11 10 10 5 04/30 Read block 278 len 12288 05/30 Write block 545 len 4096 data: 12 06/30 Read block 482 len 40960 07/30 Read block 312 len 8192 08/30 Read block 212 len 12288 09/30 Read block 191 len 8192 10/30 Write block 83 len 28672 data: 6 6 7 12 11 10 11 11/30 Write block 585 len 8192 data: 8 7 12/30 Write block 9 len 12288 data: 5 6 6 13/30 Write block 425 len 12288 data: 9 8 9 14/30 Flush 15/30 Write block 169 len 20480 data: 6 7 7 6 6 16/30 Read block 20 len 16384 17/30 Read block 272 len 4096 18/30 Read block 147 len 36864 19/30 Write block 436 len 40960 data: 6 5 5 4 4 6 5 6 8 8 20/30 Write block 429 len 36864 data: 8 8 7 8 9 7 7 7 6 21/30 Write block 400 len 12288 data: 10 12 11 22/30 Read block 32 len 16384 23/30 Flush 24/30 Write block 587 len 20480 data: 6 5 5 4 3 25/30 Read block 286 len 40960 26/30 Read block 490 len 32768 27/30 Read block 516 len 40960 28/30 Write block 209 len 36864 data: 6 6 5 4 7 7 7 10 12 29/30 Read block 218 len 40960 30/30 Write block 265 len 36864 data: 7 7 8 8 8 7 6 6 6 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 5 Done Done Done false 25 Acked 1024 Read 10 Done Done Done false 26 Acked 1025 Read 8 Done Done Done false 27 Acked 1026 Read 10 Done Done Done false 28 Acked 1027 Write 9 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:31:23.119287291Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1542} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 156 156 156 F F F 1 020-039 79 79 79 140 140 140 F F F 2 040-059 90 90 90 156 156 156 F F F 3 060-079 79 79 79 140 140 140 F F F 4 080-099 90 90 90 156 156 156 F F F 5 100-119 88 88 88 154 154 154 F F F 6 120-139 88 88 88 154 154 154 F F F 7 140-159 87 87 87 152 152 152 F F F 8 160-179 90 90 90 157 157 157 F F F 9 180-199 82 82 82 146 146 146 F F F 10 200-219 88 88 88 153 153 153 T T T 11 220-239 87 87 87 152 152 152 F F F 12 240-259 84 84 84 147 147 147 F F F 13 260-279 88 88 88 153 153 153 T T T 14 280-299 89 89 89 155 155 155 F F F 15 300-319 88 88 88 153 153 153 F F F 16 320-339 88 88 88 153 153 153 F F F 17 340-359 80 80 80 144 144 144 F F F 18 360-379 87 87 87 152 152 152 F F F 19 380-399 87 87 87 152 152 152 F F F 20 400-419 90 90 90 157 157 157 F F F 21 420-439 90 90 90 157 157 157 F F F 22 440-459 90 90 90 157 157 157 F F F 23 460-479 88 88 88 153 153 153 F F F 24 480-499 89 89 89 155 155 155 F F F 25 500-519 87 87 87 152 152 152 F F F 26 520-539 90 90 90 156 156 156 F F F 27 540-559 90 90 90 156 156 156 F F F 28 560-579 86 86 86 151 151 151 F F F 29 580-599 90 90 90 156 156 156 T T T Max gen: 90, Max flush: 157 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 156 156 156 F F F 1 020-039 79 79 79 140 140 140 F F F 2 040-059 90 90 90 156 156 156 F F F 3 060-079 79 79 79 140 140 140 F F F 4 080-099 90 90 90 156 156 156 F F F 5 100-119 88 88 88 154 154 154 F F F 6 120-139 88 88 88 154 154 154 F F F 7 140-159 87 87 87 152 152 152 F F F 8 160-179 90 90 90 157 157 157 F F F 9 180-199 82 82 82 146 146 146 F F F 10 200-219 91 91 91 158 158 158 F F F 11 220-239 87 87 87 152 152 152 F F F 12 240-259 84 84 84 147 147 147 F F F 13 260-279 91 91 91 158 158 158 F F F 14 280-299 89 89 89 155 155 155 F F F 15 300-319 88 88 88 153 153 153 F F F 16 320-339 88 88 88 153 153 153 F F F 17 340-359 80 80 80 144 144 144 F F F 18 360-379 87 87 87 152 152 152 F F F 19 380-399 87 87 87 152 152 152 F F F 20 400-419 90 90 90 157 157 157 F F F 21 420-439 90 90 90 157 157 157 F F F 22 440-459 90 90 90 157 157 157 F F F 23 460-479 88 88 88 153 153 153 F F F 24 480-499 89 89 89 155 155 155 F F F 25 500-519 87 87 87 152 152 152 F F F 26 520-539 90 90 90 156 156 156 F F F 27 540-559 90 90 90 156 156 156 F F F 28 560-579 86 86 86 151 151 151 F F F 29 580-599 91 91 91 158 158 158 F F F Max gen: 91, Max flush: 158 Begin loop 46 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.504725053Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.505014317Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.505034016Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.505049555Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550} {"msg":"Crucible fd129664-15b1-4d03-b54c-d7b54cb61de8 has session id: e16ec013-563d-4049-8a7c-fd93926d69f5","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.505273322Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"Upstairs opts: Upstairs UUID: fd129664-15b1-4d03-b54c-d7b54cb61de8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.505322159Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"Crucible stats registered with UUID: fd129664-15b1-4d03-b54c-d7b54cb61de8","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.505347178Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.536981207Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550} {"msg":"fd129664-15b1-4d03-b54c-d7b54cb61de8 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.537011715Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.537030404Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"client":"0","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.537058292Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"client":"0","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.537076341Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"client":"1","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.53709305Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"client":"1","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.537112249Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"client":"2","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.537133508Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"client":"2","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.537150137Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.537179245Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.537212464Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"ds_connection connected from Ok(127.0.0.1:45250)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.537244312Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"ds_connection connected from Ok(127.0.0.1:54293)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.537432791Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"ds_connection connected from Ok(127.0.0.1:47454)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.537469589Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.537991319Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"client":"1","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.538025647Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"client":"1","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.538054916Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"client":"2","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.538082844Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"client":"2","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.538130072Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"client":"0","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.53814898Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"client":"0","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.538565317Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"client":"1","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.538603325Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"client":"2","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.538856211Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"client":"0","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.539168963Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"client":"1","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.539196601Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"client":"1","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.53921114Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.53922391Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"client":"2","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.539237839Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"client":"2","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.539255908Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.539368382Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"client":"0","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.53939558Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"client":"0","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"[0]R flush_numbers[0..12]: [156, 140, 156, 140, 156, 154, 154, 152, 157, 146, 158, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.539411639Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"":"downstairs","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"[0]R generation[0..12]: [90, 79, 90, 79, 90, 88, 88, 87, 90, 82, 91, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.539426338Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"":"downstairs","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.539440797Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"":"downstairs","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"[1]R flush_numbers[0..12]: [156, 140, 156, 140, 156, 154, 154, 152, 157, 146, 158, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.539455446Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"":"downstairs","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"[1]R generation[0..12]: [90, 79, 90, 79, 90, 88, 88, 87, 90, 82, 91, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.539469246Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"":"downstairs","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.539483425Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"":"downstairs","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"[2]R flush_numbers[0..12]: [156, 140, 156, 140, 156, 154, 154, 152, 157, 146, 158, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.539500484Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"":"downstairs","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"[2]R generation[0..12]: [90, 79, 90, 79, 90, 88, 88, 87, 90, 82, 91, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.539514293Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"":"downstairs","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.539528192Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"":"downstairs","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.539541012Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"":"downstairs","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.539554071Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"":"downstairs","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"Next flush: 159","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.53956685Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"":"downstairs","Wait for a query_work_queue command to finish before sending IO ":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.539583099Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"":"downstairs","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.539596369Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.539608188Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"client":"0","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.539621317Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"client":"1","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.539636366Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"client":"2","":"downstairs","session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.539655955Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.539668874Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"fd129664-15b1-4d03-b54c-d7b54cb61de8 is now active with session: e16ec013-563d-4049-8a7c-fd93926d69f5","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.539681714Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.539693703Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550,"session_id":"e16ec013-563d-4049-8a7c-fd93926d69f5"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:27.539708172Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 448 len 40960 data: 8 9 8 7 6 6 7 8 9 9 02/30 Read block 426 len 32768 03/30 Write block 556 len 20480 data: 6 5 6 5 6 04/30 Read block 392 len 24576 05/30 Write block 249 len 28672 data: 7 5 4 5 4 5 6 06/30 Read block 405 len 40960 07/30 Read block 519 len 16384 08/30 Write block 11 len 12288 data: 7 7 7 09/30 Read block 107 len 4096 10/30 Read block 450 len 4096 11/30 Read block 516 len 16384 12/30 Read block 101 len 32768 13/30 Read block 295 len 16384 14/30 Read block 518 len 36864 15/30 Read block 55 len 40960 16/30 Read block 8 len 32768 17/30 Read block 520 len 24576 18/30 Write block 259 len 12288 data: 5 6 7 19/30 Write block 212 len 12288 data: 5 8 8 20/30 Read block 453 len 16384 21/30 Write block 45 len 36864 data: 4 6 8 9 9 11 9 10 10 22/30 Write block 325 len 16384 data: 7 6 8 9 23/30 Write block 445 len 12288 data: 9 10 10 24/30 Read block 220 len 28672 25/30 Read block 8 len 12288 26/30 Read block 190 len 36864 27/30 Write block 65 len 16384 data: 7 9 9 9 28/30 Read block 479 len 16384 29/30 Read block 542 len 36864 30/30 Read block 231 len 32768 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Write 10 Done Done Done false 2 Acked 1001 Read 8 Done Done Done false 3 Acked 1002 Write 5 Done Done Done false 4 Acked 1003 Read 6 Done Done Done false 5 Acked 1004 Write 7 Done Done Done false 6 Acked 1005 Read 10 Done Done Done false 7 Acked 1006 Read 4 Done Done Done false 8 Acked 1007 Write 3 Done Done Done false 9 Acked 1008 Read 1 Done Done Done false 10 Acked 1009 Read 1 Done Done Done false 11 Acked 1010 Read 4 Done Done Done false 12 Acked 1011 Read 8 Done Done Done false 13 Acked 1012 Read 4 Done Done Done false 14 Acked 1013 Read 9 Done Done Done false 15 Acked 1014 Read 10 Done Done Done false 16 Acked 1015 Read 8 Done Done Done false 17 Acked 1016 Read 6 Done Done Done false 18 Acked 1017 Write 3 Done Done Done false 19 Acked 1018 Write 3 Done Done Done false 20 Acked 1019 Read 4 Done Done Done false 21 Acked 1020 Write 9 Done Done Done false 22 Acked 1021 Write 4 Done Done Done false 23 Acked 1022 Write 3 Done Done Done false 24 Acked 1023 Read 7 Done Done Done false 25 Acked 1024 Read 3 Done Done Done false 26 Acked 1025 Read 9 Done Done Done false 27 Acked 1026 Write 4 Done Done Done false 28 Acked 1027 Read 4 Done Done Done false 29 Acked 1028 Read 9 Done Done Done false 30 Acked 1029 Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 30 29 29 88 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:31:27.560467096Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1550} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 156 156 156 T T T 1 020-039 79 79 79 140 140 140 F F F 2 040-059 90 90 90 156 156 156 T T T 3 060-079 79 79 79 140 140 140 T T T 4 080-099 90 90 90 156 156 156 F F F 5 100-119 88 88 88 154 154 154 F F F 6 120-139 88 88 88 154 154 154 F F F 7 140-159 87 87 87 152 152 152 F F F 8 160-179 90 90 90 157 157 157 F F F 9 180-199 82 82 82 146 146 146 F F F 10 200-219 91 91 91 158 158 158 T T T 11 220-239 87 87 87 152 152 152 F F F 12 240-259 84 84 84 147 147 147 T T T 13 260-279 91 91 91 158 158 158 T T T 14 280-299 89 89 89 155 155 155 F F F 15 300-319 88 88 88 153 153 153 F F F 16 320-339 88 88 88 153 153 153 T T T 17 340-359 80 80 80 144 144 144 F F F 18 360-379 87 87 87 152 152 152 F F F 19 380-399 87 87 87 152 152 152 F F F 20 400-419 90 90 90 157 157 157 F F F 21 420-439 90 90 90 157 157 157 F F F 22 440-459 90 90 90 157 157 157 T T T 23 460-479 88 88 88 153 153 153 F F F 24 480-499 89 89 89 155 155 155 F F F 25 500-519 87 87 87 152 152 152 F F F 26 520-539 90 90 90 156 156 156 F F F 27 540-559 90 90 90 156 156 156 T T T 28 560-579 86 86 86 151 151 151 T T T 29 580-599 91 91 91 158 158 158 F F F Max gen: 91, Max flush: 158 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 159 159 159 F F F 1 020-039 79 79 79 140 140 140 F F F 2 040-059 92 92 92 159 159 159 F F F 3 060-079 92 92 92 159 159 159 F F F 4 080-099 90 90 90 156 156 156 F F F 5 100-119 88 88 88 154 154 154 F F F 6 120-139 88 88 88 154 154 154 F F F 7 140-159 87 87 87 152 152 152 F F F 8 160-179 90 90 90 157 157 157 F F F 9 180-199 82 82 82 146 146 146 F F F 10 200-219 92 92 92 159 159 159 F F F 11 220-239 87 87 87 152 152 152 F F F 12 240-259 92 92 92 159 159 159 F F F 13 260-279 92 92 92 159 159 159 F F F 14 280-299 89 89 89 155 155 155 F F F 15 300-319 88 88 88 153 153 153 F F F 16 320-339 92 92 92 159 159 159 F F F 17 340-359 80 80 80 144 144 144 F F F 18 360-379 87 87 87 152 152 152 F F F 19 380-399 87 87 87 152 152 152 F F F 20 400-419 90 90 90 157 157 157 F F F 21 420-439 90 90 90 157 157 157 F F F 22 440-459 92 92 92 159 159 159 F F F 23 460-479 88 88 88 153 153 153 F F F 24 480-499 89 89 89 155 155 155 F F F 25 500-519 87 87 87 152 152 152 F F F 26 520-539 90 90 90 156 156 156 F F F 27 540-559 92 92 92 159 159 159 F F F 28 560-579 92 92 92 159 159 159 F F F 29 580-599 91 91 91 158 158 158 F F F Max gen: 92, Max flush: 159 Begin loop 47 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.036485299Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.03681173Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.036848228Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.036875807Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558} {"msg":"Crucible 93d921ee-f182-443d-9416-a6699bddd917 has session id: cdcb1e67-a9cd-4e6e-8fed-b5153d04a694","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.036973021Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"Upstairs opts: Upstairs UUID: 93d921ee-f182-443d-9416-a6699bddd917, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.037025808Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"Crucible stats registered with UUID: 93d921ee-f182-443d-9416-a6699bddd917","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.037050297Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.068169994Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558} {"msg":"93d921ee-f182-443d-9416-a6699bddd917 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.068200013Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.068216282Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"client":"0","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.06824781Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"client":"0","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.068275198Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"client":"1","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.068309496Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"client":"1","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.068343445Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"client":"2","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.068371813Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"client":"2","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.068398171Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.06842652Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"":"io task","client":"0","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.068451288Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"ds_connection connected from Ok(127.0.0.1:62742)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.068478227Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"ds_connection connected from Ok(127.0.0.1:47081)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.068505005Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"":"io task","client":"0","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"ds_connection connected from Ok(127.0.0.1:64176)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.068532154Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.069262353Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"client":"1","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.06929838Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"client":"1","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.069322849Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"client":"2","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.069346788Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"client":"2","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.069489349Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"client":"0","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.069524467Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"client":"0","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.069930444Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"client":"2","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.069966513Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"client":"1","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.07001138Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"client":"0","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.07054366Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"client":"1","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.070570748Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"client":"1","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.070587078Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.070599947Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"client":"0","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.070613356Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"client":"0","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.070626655Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.070638755Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"client":"2","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.070652204Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"client":"2","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"[0]R flush_numbers[0..12]: [159, 140, 159, 159, 156, 154, 154, 152, 157, 146, 159, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.070666093Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"":"downstairs","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"[0]R generation[0..12]: [92, 79, 92, 92, 90, 88, 88, 87, 90, 82, 92, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.070679882Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"":"downstairs","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.070694011Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"":"downstairs","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"[1]R flush_numbers[0..12]: [159, 140, 159, 159, 156, 154, 154, 152, 157, 146, 159, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.070707811Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"":"downstairs","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"[1]R generation[0..12]: [92, 79, 92, 92, 90, 88, 88, 87, 90, 82, 92, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.07072141Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"":"downstairs","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.070739229Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"":"downstairs","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"[2]R flush_numbers[0..12]: [159, 140, 159, 159, 156, 154, 154, 152, 157, 146, 159, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.070753338Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"":"downstairs","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"[2]R generation[0..12]: [92, 79, 92, 92, 90, 88, 88, 87, 90, 82, 92, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.070766987Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"":"downstairs","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.070780816Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"":"downstairs","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"Max found gen is 93","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.070793716Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"":"downstairs","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"Wait for a query_work_queue command to finish before sending IO Generation requested: 94 >= found:93","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.070809665Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"":"downstairs","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"Next flush: 160","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.070822614Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"":"downstairs","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.070836413Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"":"downstairs","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.070849703Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.070861812Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"client":"0","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.070874951Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"client":"1","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.07088786Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"client":"2","":"downstairs","session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-09-26T14:31:32.07090101Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.070918389Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"93d921ee-f182-443d-9416-a6699bddd917 is now active with session: cdcb1e67-a9cd-4e6e-8fed-b5153d04a694","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.070931778Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.070946427Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558,"session_id":"cdcb1e67-a9cd-4e6e-8fed-b5153d04a694"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:32.070958936Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 472 len 28672 data: 6 6 6 7 8 6 7 02/30 Write block 110 len 36864 data: 7 11 13 14 12 11 11 10 9 03/30 Read block 221 len 24576 04/30 Read block 473 len 24576 05/30 Read block 195 len 24576 06/30 Read block 481 len 12288 07/30 Write block 545 len 20480 data: 13 12 11 11 9 08/30 Read block 531 len 36864 09/30 Read block 163 len 40960 10/30 Flush 11/30 Write block 147 len 28672 data: 7 3 3 3 6 6 6 12/30 Flush 13/30 Write block 313 len 32768 data: 11 12 10 11 11 7 10 9 14/30 Write block 482 len 16384 data: 7 7 6 7 15/30 Read block 248 len 32768 16/30 Read block 26 len 8192 17/30 Read block 173 len 28672 18/30 Read block 452 len 8192 19/30 Read block 555 len 24576 20/30 Write block 360 len 24576 data: 9 7 8 7 8 6 21/30 Write block 266 len 16384 data: 8 9 9 9 22/30 Read block 431 len 40960 23/30 Read block 461 len 20480 24/30 Read block 30 len 28672 25/30 Write block 315 len 16384 data: 11 12 12 8 26/30 Read block 565 len 16384 27/30 Write block 528 len 40960 data: 8 7 7 8 9 10 9 8 8 9 28/30 Read block 572 len 24576 29/30 Read block 395 len 24576 30/30 Read block 500 len 20480 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 13 Acked 1012 Write 8 Done Done Done false 14 Acked 1013 Write 4 Done Done Done false 15 Acked 1014 Read 8 Done Done Done false 16 Acked 1015 Read 2 Done Done Done false 17 Acked 1016 Read 7 Done Done Done false 18 Acked 1017 Read 2 Done Done Done false 19 Acked 1018 Read 6 Done Done Done false 20 Acked 1019 Write 6 Done Done Done false 21 Acked 1020 Write 4 Done Done Done false 22 Acked 1021 Read 10 Done Done Done false 23 Acked 1022 Read 5 Done Done Done false 24 Acked 1023 Read 7 Done Done Done false 25 Acked 1024 Write 4 Done Done Done false 26 Acked 1025 Read 4 Done Done Done false 27 Acked 1026 Write 10 Done Done Done false 28 Acked 1027 Read 6 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Read 5 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 18 17 18 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:31:32.095744603Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1558} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 159 159 159 F F F 1 020-039 79 79 79 140 140 140 F F F 2 040-059 92 92 92 159 159 159 F F F 3 060-079 92 92 92 159 159 159 F F F 4 080-099 90 90 90 156 156 156 F F F 5 100-119 94 94 94 160 160 160 F F F 6 120-139 88 88 88 154 154 154 F F F 7 140-159 94 94 94 161 161 161 F F F 8 160-179 90 90 90 157 157 157 F F F 9 180-199 82 82 82 146 146 146 F F F 10 200-219 92 92 92 159 159 159 F F F 11 220-239 87 87 87 152 152 152 F F F 12 240-259 92 92 92 159 159 159 F F F 13 260-279 92 92 92 159 159 159 T T T 14 280-299 89 89 89 155 155 155 F F F 15 300-319 88 88 88 153 153 153 T T T 16 320-339 92 92 92 159 159 159 T T T 17 340-359 80 80 80 144 144 144 F F F 18 360-379 87 87 87 152 152 152 T T T 19 380-399 87 87 87 152 152 152 F F F 20 400-419 90 90 90 157 157 157 F F F 21 420-439 90 90 90 157 157 157 F F F 22 440-459 92 92 92 159 159 159 F F F 23 460-479 94 94 94 160 160 160 F F F 24 480-499 89 89 89 155 155 155 T T T 25 500-519 87 87 87 152 152 152 F F F 26 520-539 90 90 90 156 156 156 T T T 27 540-559 94 94 94 160 160 160 F F F 28 560-579 92 92 92 159 159 159 F F F 29 580-599 91 91 91 158 158 158 F F F Max gen: 94, Max flush: 161 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 159 159 159 F F F 1 020-039 79 79 79 140 140 140 F F F 2 040-059 92 92 92 159 159 159 F F F 3 060-079 92 92 92 159 159 159 F F F 4 080-099 90 90 90 156 156 156 F F F 5 100-119 94 94 94 160 160 160 F F F 6 120-139 88 88 88 154 154 154 F F F 7 140-159 94 94 94 161 161 161 F F F 8 160-179 90 90 90 157 157 157 F F F 9 180-199 82 82 82 146 146 146 F F F 10 200-219 92 92 92 159 159 159 F F F 11 220-239 87 87 87 152 152 152 F F F 12 240-259 92 92 92 159 159 159 F F F 13 260-279 95 95 95 162 162 162 F F F 14 280-299 89 89 89 155 155 155 F F F 15 300-319 95 95 95 162 162 162 F F F 16 320-339 95 95 95 162 162 162 F F F 17 340-359 80 80 80 144 144 144 F F F 18 360-379 95 95 95 162 162 162 F F F 19 380-399 87 87 87 152 152 152 F F F 20 400-419 90 90 90 157 157 157 F F F 21 420-439 90 90 90 157 157 157 F F F 22 440-459 92 92 92 159 159 159 F F F 23 460-479 94 94 94 160 160 160 F F F 24 480-499 95 95 95 162 162 162 F F F 25 500-519 87 87 87 152 152 152 F F F 26 520-539 95 95 95 162 162 162 F F F 27 540-559 94 94 94 160 160 160 F F F 28 560-579 92 92 92 159 159 159 F F F 29 580-599 91 91 91 158 158 158 F F F Max gen: 95, Max flush: 162 Begin loop 48 Downstairs to restart: 2 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.491065177Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.491374329Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.491406657Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.491435186Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566} {"msg":"Crucible af91e6ba-e078-420a-a728-eab0a55f2805 has session id: e2e2fd48-223e-42c7-83fc-b911f63353e6","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.49153204Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"Upstairs opts: Upstairs UUID: af91e6ba-e078-420a-a728-eab0a55f2805, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.491587287Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"Crucible stats registered with UUID: af91e6ba-e078-420a-a728-eab0a55f2805","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.491609626Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.522896974Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566} {"msg":"af91e6ba-e078-420a-a728-eab0a55f2805 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.522932352Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.52295625Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"client":"0","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.523000658Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"client":"0","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.523036026Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"client":"1","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.523071114Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"client":"1","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.523104332Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"client":"2","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.5231391Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"client":"2","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.523165279Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"":"io task","client":"2","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.523200597Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"":"io task","client":"0","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.523235515Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:61213)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.523452482Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"":"io task","client":"2","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:46775)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.52348562Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:53570)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.523566656Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"":"io task","client":"0","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.524071397Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"client":"0","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.524105735Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"client":"0","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.524135594Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"client":"1","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.524165802Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"client":"1","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.524225029Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"client":"2","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.524253427Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"client":"2","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.5247352Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"client":"0","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.524776278Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"client":"1","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.524894181Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"client":"2","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.525360564Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"client":"1","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.525391812Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"client":"1","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.525406902Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.525466358Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"client":"0","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.525505056Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"client":"0","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.525532355Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.525553174Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"client":"2","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.525575972Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"client":"2","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"[0]R flush_numbers[0..12]: [159, 140, 159, 159, 156, 160, 154, 161, 157, 146, 159, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.525600511Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"":"downstairs","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"[0]R generation[0..12]: [92, 79, 92, 92, 90, 94, 88, 94, 90, 82, 92, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.525625979Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"":"downstairs","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.525650598Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"":"downstairs","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"[1]R flush_numbers[0..12]: [159, 140, 159, 159, 156, 160, 154, 161, 157, 146, 159, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.525673616Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"":"downstairs","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"[1]R generation[0..12]: [92, 79, 92, 92, 90, 94, 88, 94, 90, 82, 92, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.525696785Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"":"downstairs","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.525723634Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"":"downstairs","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"[2]R flush_numbers[0..12]: [159, 140, 159, 159, 156, 160, 154, 161, 157, 146, 159, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.525749282Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"":"downstairs","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"[2]R generation[0..12]: [92, 79, 92, 92, 90, 94, 88, 94, 90, 82, 92, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.525777061Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"":"downstairs","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.525805009Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"":"downstairs","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg"Wait for a query_work_queue command to finish before sending IO :"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.525835458Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"":"downstairs","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.525862246Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"":"downstairs","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"Next flush: 163","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.525888884Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"":"downstairs","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.525915843Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"":"downstairs","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.525942521Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2024-09-26T14:31:36.52596644Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"client":"0","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.525998748Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"client":"1","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.526017947Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"client":"2","":"downstairs","session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.526031466Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.526043736Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"af91e6ba-e078-420a-a728-eab0a55f2805 is now active with session: e2e2fd48-223e-42c7-83fc-b911f63353e6","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.526056595Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.526068624Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566,"session_id":"e2e2fd48-223e-42c7-83fc-b911f63353e6"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:36.526080964Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 214 len 20480 data: 9 8 11 13 11 02/30 Read block 88 len 4096 03/30 Read block 445 len 8192 04/30 Write block 40 len 36864 data: 5 4 4 5 4 5 7 9 10 05/30 Write block 440 len 16384 data: 5 7 6 7 06/30 Read block 571 len 36864 07/30 Read block 508 len 4096 08/30 Read block 467 len 28672 09/30 Read block 428 len 8192 10/30 Write block 347 len 40960 data: 9 9 7 6 8 9 11 9 9 7 11/30 Write block 357 len 28672 data: 9 10 9 10 8 9 8 12/30 Write block 489 len 40960 data: 7 7 7 8 8 10 10 9 9 9 13/30 Write block 587 len 16384 data: 7 6 6 5 14/30 Read block 288 len 8192 15/30 Write block 101 len 24576 data: 9 7 8 8 8 7 16/30 Read block 389 len 16384 17/30 Read block 339 len 24576 18/30 Flush 19/30 Write block 82 len 28672 data: 9 7 7 8 13 12 11 20/30 Read block 130 len 28672 21/30 Read block 145 len 4096 22/30 Write block 327 len 24576 data: 9 10 8 7 8 6 23/30 Write block 471 len 16384 data: 7 7 7 7 24/30 Read block 580 len 40960 25/30 Read block 96 len 12288 26/30 Write block 516 len 40960 data: 4 6 8 9 10 8 6 10 10 10 27/30 Write block 255 len 32768 data: 7 6 3 4 6 7 8 8 28/30 Read block 406 len 16384 29/30 Read block 121 len 4096 30/30 Write block 143 len 4096 data: 4 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 19 Acked 1018 Write 7 Done Done Done false 20 Acked 1019 Read 7 Done Done Done false 21 Acked 1020 Read 1 Done Done Done false 22 Acked 1021 Write 6 Done Done Done false 23 Acked 1022 Write 4 Done Done Done false 24 Acked 1023 Read 10 Done Done Done false 25 Acked 1024 Read 3 Done Done Done false 26 Acked 1025 Write 10 Done Done Done false 27 Acked 1026 Write 8 Done Done Done false 28 Acked 1027 Read 4 Done Done Done false 29 Acked 1028 Read 1 Done Done Done false 30 Acked 1029 Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:31:36.548602658Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1566} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 159 159 159 F F F 1 020-039 79 79 79 140 140 140 F F F 2 040-059 96 96 96 163 163 163 F F F 3 060-079 92 92 92 159 159 159 F F F 4 080-099 90 90 90 156 156 156 T T T 5 100-119 96 96 96 163 163 163 F F F 6 120-139 88 88 88 154 154 154 F F F 7 140-159 94 94 94 161 161 161 T T T 8 160-179 90 90 90 157 157 157 F F F 9 180-199 82 82 82 146 146 146 F F F 10 200-219 96 96 96 163 163 163 F F F 11 220-239 87 87 87 152 152 152 F F F 12 240-259 92 92 92 159 159 159 T T T 13 260-279 95 95 95 162 162 162 T T T 14 280-299 89 89 89 155 155 155 F F F 15 300-319 95 95 95 162 162 162 F F F 16 320-339 95 95 95 162 162 162 T T T 17 340-359 96 96 96 163 163 163 F F F 18 360-379 96 96 96 163 163 163 F F F 19 380-399 87 87 87 152 152 152 F F F 20 400-419 90 90 90 157 157 157 F F F 21 420-439 90 90 90 157 157 157 F F F 22 440-459 96 96 96 163 163 163 F F F 23 460-479 94 94 94 160 160 160 T T T 24 480-499 96 96 96 163 163 163 F F F 25 500-519 87 87 87 152 152 152 T T T 26 520-539 95 95 95 162 162 162 T T T 27 540-559 94 94 94 160 160 160 F F F 28 560-579 92 92 92 159 159 159 F F F 29 580-599 96 96 96 163 163 163 F F F Max gen: 96, Max flush: 163 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 159 159 159 F F F 1 020-039 79 79 79 140 140 140 F F F 2 040-059 96 96 96 163 163 163 F F F 3 060-079 92 92 92 159 159 159 F F F 4 080-099 97 97 97 164 164 164 F F F 5 100-119 96 96 96 163 163 163 F F F 6 120-139 88 88 88 154 154 154 F F F 7 140-159 97 97 97 164 164 164 F F F 8 160-179 90 90 90 157 157 157 F F F 9 180-199 82 82 82 146 146 146 F F F 10 200-219 96 96 96 163 163 163 F F F 11 220-239 87 87 87 152 152 152 F F F 12 240-259 97 97 97 164 164 164 F F F 13 260-279 97 97 97 164 164 164 F F F 14 280-299 89 89 89 155 155 155 F F F 15 300-319 95 95 95 162 162 162 F F F 16 320-339 97 97 97 164 164 164 F F F 17 340-359 96 96 96 163 163 163 F F F 18 360-379 96 96 96 163 163 163 F F F 19 380-399 87 87 87 152 152 152 F F F 20 400-419 90 90 90 157 157 157 F F F 21 420-439 90 90 90 157 157 157 F F F 22 440-459 96 96 96 163 163 163 F F F 23 460-479 97 97 97 164 164 164 F F F 24 480-499 96 96 96 163 163 163 F F F 25 500-519 97 97 97 164 164 164 F F F 26 520-539 97 97 97 164 164 164 F F F 27 540-559 94 94 94 160 160 160 F F F 28 560-579 92 92 92 159 159 159 F F F 29 580-599 96 96 96 163 163 163 F F F Max gen: 97, Max flush: 164 Begin loop 49 Downstairs to restart: 2 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.935975271Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.936261844Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.936286993Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.936311262Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574} {"msg":"Crucible 025bb9b5-52f6-4165-ae6a-7995100a2eca has session id: 4f60aba9-fd56-412b-964d-fd31af238bb1","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.936458433Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"Upstairs opts: Upstairs UUID: 025bb9b5-52f6-4165-ae6a-7995100a2eca, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.936525189Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"Crucible stats registered with UUID: 025bb9b5-52f6-4165-ae6a-7995100a2eca","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.936550158Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.967646427Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574} {"msg":"025bb9b5-52f6-4165-ae6a-7995100a2eca active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.967679155Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.967697214Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"client":"0","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.967724703Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"client":"0","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.967742012Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"client":"1","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.967758801Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"client":"1","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.96777524Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"client":"2","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.967795619Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"client":"2","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.967814368Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"":"io task","client":"2","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.967844586Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.967876324Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"ds_connection connected from Ok(127.0.0.1:58305)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.967909102Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"ds_connection connected from Ok(127.0.0.1:36748)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.968195876Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"ds_connection connected from Ok(127.0.0.1:58484)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.968231714Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"":"io task","client":"2","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.968694318Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"client":"0","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.968721796Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"client":"0","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.968868018Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"client":"1","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.968887337Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"client":"1","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.968932554Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"client":"2","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.968959793Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"client":"2","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.969337791Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"client":"0","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.969371159Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"client":"1","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.969663303Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"client":"2","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.96989043Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"client":"1","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.969925238Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"client":"1","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.969941847Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.969954536Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"client":"0","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.969971795Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"client":"0","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.969985415Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.970152645Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"client":"2","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.970178154Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"client":"2","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"[0]R flush_numbers[0..12]: [159, 140, 163, 159, 164, 163, 154, 164, 157, 146, 163, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.970194623Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"":"downstairs","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"[0]R generation[0..12]: [92, 79, 96, 92, 97, 96, 88, 97, 90, 82, 96, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.970209122Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"":"downstairs","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.970223401Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"":"downstairs","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"[1]R flush_numbers[0..12]: [159, 140, 163, 159, 164, 163, 154, 164, 157, 146, 163, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.97023738Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"":"downstairs","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"[1]R generation[0..12]: [92, 79, 96, 92, 97, 96, 88, 97, 90, 82, 96, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.97025119Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"":"downstairs","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.970265229Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"":"downstairs","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"[2]R flush_numbers[0..12]: [159, 140, 163, 159, 164, 163, 154, 164, 157, 146, 163, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.970279108Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"":"downstairs","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"[2]R generation[0..12]: [92, 79, 96, 92, 97, 96, 88, 97, 90, 82, 96, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.970292657Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"":"downstairs","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.970306656Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"":"downstairs","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.970319676Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"":"downstairs","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.970337195Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"":"downstairs","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"Next flush: 165","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.970350724Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"":"downstairs","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-26T14:31:40.970363563Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"":"downstairs","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.970376983Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.970389122Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"client":"0","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.970402601Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"client":"1","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.9704159Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"client":"2","":"downstairs","session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.970429169Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.970441329Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"025bb9b5-52f6-4165-ae6a-7995100a2eca is now active with session: 4f60aba9-fd56-412b-964d-fd31af238bb1","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.970454128Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.970465897Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574,"session_id":"4f60aba9-fd56-412b-964d-fd31af238bb1"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:40.970477957Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 1 len 20480 data: 7 6 8 7 6 02/30 Read block 34 len 8192 03/30 Read block 587 len 36864 04/30 Read block 167 len 4096 05/30 Write block 96 len 40960 data: 3 3 4 5 7 10 8 9 9 9 06/30 Read block 280 len 28672 07/30 Flush 08/30 Write block 435 len 24576 data: 8 8 7 6 5 6 09/30 Read block 318 len 16384 10/30 Write block 285 len 4096 data: 10 11/30 Read block 444 len 20480 12/30 Read block 217 len 32768 13/30 Read block 370 len 32768 14/30 Write block 169 len 20480 data: 7 8 8 7 7 15/30 Write block 572 len 8192 data: 9 10 16/30 Flush 17/30 Write block 345 len 4096 data: 7 18/30 Write block 215 len 4096 data: 9 19/30 Read block 369 len 32768 20/30 Write block 254 len 4096 data: 6 21/30 Read block 562 len 40960 22/30 Read block 256 len 36864 23/30 Read block 208 len 28672 24/30 Read block 274 len 28672 25/30 Read block 393 len 32768 26/30 Read block 84 len 24576 27/30 Read block 144 len 20480 28/30 Read block 516 len 24576 29/30 Read block 469 len 12288 30/30 Write block 176 len 20480 data: 6 5 5 7 9 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 17 Acked 1016 Write 1 Done Done Done false 18 Acked 1017 Write 1 Done Done Done false 19 Acked 1018 Read 8 Done Done Done false 20 Acked 1019 Write 1 Done Done Done false 21 Acked 1020 Read 10 Done Done Done false 22 Acked 1021 Read 9 Done Done Done false 23 Acked 1022 Read 7 Done Done Done false 24 Acked 1023 Read 7 Done Done Done false 25 Acked 1024 Read 8 Done Done Done false 26 Acked 1025 Read 6 Done Done Done false 27 Acked 1026 Read 5 Done Done Done false 28 Acked 1027 Read 6 Done Done Done false 29 Acked 1028 Read 3 Done Done Done false 30 Acked 1029 Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:31:40.994051842Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1574} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 165 165 165 F F F 1 020-039 79 79 79 140 140 140 F F F 2 040-059 96 96 96 163 163 163 F F F 3 060-079 92 92 92 159 159 159 F F F 4 080-099 98 98 98 165 165 165 F F F 5 100-119 98 98 98 165 165 165 F F F 6 120-139 88 88 88 154 154 154 F F F 7 140-159 97 97 97 164 164 164 F F F 8 160-179 98 98 98 166 166 166 T T T 9 180-199 82 82 82 146 146 146 T T T 10 200-219 96 96 96 163 163 163 T T T 11 220-239 87 87 87 152 152 152 F F F 12 240-259 97 97 97 164 164 164 T T T 13 260-279 97 97 97 164 164 164 F F F 14 280-299 98 98 98 166 166 166 F F F 15 300-319 95 95 95 162 162 162 F F F 16 320-339 97 97 97 164 164 164 F F F 17 340-359 96 96 96 163 163 163 T T T 18 360-379 96 96 96 163 163 163 F F F 19 380-399 87 87 87 152 152 152 F F F 20 400-419 90 90 90 157 157 157 F F F 21 420-439 98 98 98 166 166 166 F F F 22 440-459 98 98 98 166 166 166 F F F 23 460-479 97 97 97 164 164 164 F F F 24 480-499 96 96 96 163 163 163 F F F 25 500-519 97 97 97 164 164 164 F F F 26 520-539 97 97 97 164 164 164 F F F 27 540-559 94 94 94 160 160 160 F F F 28 560-579 98 98 98 166 166 166 F F F 29 580-599 96 96 96 163 163 163 F F F Max gen: 98, Max flush: 166 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 165 165 165 F F F 1 020-039 79 79 79 140 140 140 F F F 2 040-059 96 96 96 163 163 163 F F F 3 060-079 92 92 92 159 159 159 F F F 4 080-099 98 98 98 165 165 165 F F F 5 100-119 98 98 98 165 165 165 F F F 6 120-139 88 88 88 154 154 154 F F F 7 140-159 97 97 97 164 164 164 F F F 8 160-179 99 99 99 167 167 167 F F F 9 180-199 99 99 99 167 167 167 F F F 10 200-219 99 99 99 167 167 167 F F F 11 220-239 87 87 87 152 152 152 F F F 12 240-259 99 99 99 167 167 167 F F F 13 260-279 97 97 97 164 164 164 F F F 14 280-299 98 98 98 166 166 166 F F F 15 300-319 95 95 95 162 162 162 F F F 16 320-339 97 97 97 164 164 164 F F F 17 340-359 99 99 99 167 167 167 F F F 18 360-379 96 96 96 163 163 163 F F F 19 380-399 87 87 87 152 152 152 F F F 20 400-419 90 90 90 157 157 157 F F F 21 420-439 98 98 98 166 166 166 F F F 22 440-459 98 98 98 166 166 166 F F F 23 460-479 97 97 97 164 164 164 F F F 24 480-499 96 96 96 163 163 163 F F F 25 500-519 97 97 97 164 164 164 F F F 26 520-539 97 97 97 164 164 164 F F F 27 540-559 94 94 94 160 160 160 F F F 28 560-579 98 98 98 166 166 166 F F F 29 580-599 96 96 96 163 163 163 F F F Max gen: 99, Max flush: 167 Begin loop 50 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.344899794Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.34531286Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.345353398Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.345386666Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582} {"msg":"Crucible 84d9d139-54f2-4616-b2a0-434af63e37e3 has session id: 98d60da1-4719-4819-a7b6-f8eba188cfb8","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.345569896Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"Upstairs opts: Upstairs UUID: 84d9d139-54f2-4616-b2a0-434af63e37e3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.345613903Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"Crucible stats registered with UUID: 84d9d139-54f2-4616-b2a0-434af63e37e3","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.345630533Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.37675444Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582} {"msg":"84d9d139-54f2-4616-b2a0-434af63e37e3 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.376785478Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.376802727Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"client":"0","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.376829846Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"client":"0","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.376860424Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"client":"1","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.376893362Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"client":"1","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.37692712Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"client":"2","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.376959388Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"client":"2","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.376992386Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"":"io task","client":"2","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.377025644Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"":"io task","client":"0","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.377068472Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"ds_connection connected from Ok(127.0.0.1:51866)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.37710222Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"":"io task","client":"0","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"ds_connection connected from Ok(127.0.0.1:56175)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.377133818Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"":"io task","client":"2","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"ds_connection connected from Ok(127.0.0.1:43849)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.377164607Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.377852608Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"client":"2","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.377880996Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"client":"2","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.377901155Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"client":"0","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.377915374Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"client":"0","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.378031087Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"client":"1","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.378063586Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"client":"1","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.378558788Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"client":"2","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.378599465Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"client":"0","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.378703979Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"client":"1","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.379100337Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"client":"0","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.379124685Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"client":"0","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.379138885Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.379194312Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"client":"2","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.37922504Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"client":"2","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.379239579Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.379295866Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"client":"1","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.379315195Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"client":"1","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"[0]R flush_numbers[0..12]: [165, 140, 163, 159, 165, 165, 154, 164, 167, 167, 167, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.379330014Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"":"downstairs","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"[0]R generation[0..12]: [98, 79, 96, 92, 98, 98, 88, 97, 99, 99, 99, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.379347693Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"":"downstairs","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.379362152Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"":"downstairs","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"[1]R flush_numbers[0..12]: [165, 140, 163, 159, 165, 165, 154, 164, 167, 167, 167, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.379375991Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"":"downstairs","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"[1]R generation[0..12]: [98, 79, 96, 92, 98, 98, 88, 97, 99, 99, 99, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.37938939Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"":"downstairs","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.37940338Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"":"downstairs","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"[2]R flush_numbers[0..12]: [165, 140, 163, 159, 165, 165, 154, 164, 167, 167, 167, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.379417029Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"":"downstairs","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"[2]R generation[0..12]: [98, 79, 96, 92, 98, 98, 88, 97, 99, 99, 99, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.379430468Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"":"downstairs","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.379444267Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"":"downstairs","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.379456957Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"":"downstairs","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.379469976Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"":"downstairs","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"Next flush: 168","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.379482795Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"":"downstairs","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.379496395Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"":"downstairs","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.379509384Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"Transition from WaitQuorum to Active","v":0,Wait for a query_work_queue command to finish before sending IO "name":"crucible","level":30,"time":"2024-09-26T14:31:45.379528343Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"client":"0","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.379553451Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"client":"1","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.37957887Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"client":"2","":"downstairs","session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.379603729Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.379627687Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"84d9d139-54f2-4616-b2a0-434af63e37e3 is now active with session: 98d60da1-4719-4819-a7b6-f8eba188cfb8","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-09-26T14:31:45.379651806Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.379676084Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582,"session_id":"98d60da1-4719-4819-a7b6-f8eba188cfb8"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:45.379697603Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 572 len 8192 data: 10 11 02/30 Flush 03/30 Write block 204 len 40960 data: 7 7 7 5 3 7 7 6 6 9 04/30 Read block 359 len 12288 05/30 Write block 497 len 20480 data: 10 10 10 9 10 06/30 Read block 103 len 24576 07/30 Read block 255 len 20480 08/30 Write block 336 len 8192 data: 5 3 09/30 Write block 316 len 24576 data: 13 13 9 11 10 7 10/30 Write block 418 len 20480 data: 7 6 9 9 9 11/30 Write block 349 len 28672 data: 8 7 9 10 12 10 10 12/30 Flush 13/30 Write block 174 len 12288 data: 5 5 7 14/30 Write block 586 len 36864 data: 8 8 7 7 6 4 4 4 6 15/30 Write block 321 len 40960 data: 8 6 6 7 8 7 10 11 9 8 16/30 Write block 384 len 8192 data: 6 6 17/30 Read block 112 len 40960 18/30 Read block 328 len 12288 19/30 Write block 261 len 32768 data: 9 9 9 8 8 9 10 10 20/30 Read block 250 len 36864 21/30 Read block 230 len 28672 22/30 Write block 183 len 40960 data: 11 11 10 6 7 7 5 6 7 8 23/30 Write block 28 len 4096 data: 8 24/30 Write block 562 len 4096 data: 5 25/30 Read block 61 len 12288 26/30 Read block 448 len 24576 27/30 Read block 551 len 20480 28/30 Read block 447 len 24576 29/30 Write block 60 len 28672 data: 8 7 6 8 8 8 10 30/30 Write block 340 len 24576 data: 7 6 7 7 7 8 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 13 Acked 1012 Write 3 Done Done Done false 14 Acked 1013 Write 9 Done Done Done false 15 Acked 1014 Write 10 Done Done Done false 16 Acked 1015 Write 2 Done Done Done false 17 Acked 1016 Read 10 Done Done Done false 18 Acked 1017 Read 3 Done Done Done false 19 Acked 1018 Write 8 Done Done Done false 20 Acked 1019 Read 9 Done Done Done false 21 Acked 1020 Read 7 Done Done Done false 22 Acked 1021 Write 10 Done Done Done false 23 Acked 1022 Write 1 Done Done Done false 24 Acked 1023 Write 1 Done Done Done false 25 Acked 1024 Read 3 Done Done Done false 26 Acked 1025 Read 6 Done Done Done false 27 Acked 1026 Read 5 Done Done Done false 28 Acked 1027 Read 6 Done Done Done false 29 Acked 1028 Write 7 Sent Sent Sent false 30 Acked 1029 Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:31:45.402622685Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1582} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 165 165 165 F F F 1 020-039 79 79 79 140 140 140 T T T 2 040-059 96 96 96 163 163 163 F F F 3 060-079 92 92 92 159 159 159 T T T 4 080-099 98 98 98 165 165 165 F F F 5 100-119 98 98 98 165 165 165 F F F 6 120-139 88 88 88 154 154 154 F F F 7 140-159 97 97 97 164 164 164 F F F 8 160-179 99 99 99 167 167 167 T T T 9 180-199 99 99 99 167 167 167 T T T 10 200-219 100 100 100 169 169 169 F F F 11 220-239 87 87 87 152 152 152 F F F 12 240-259 99 99 99 167 167 167 F F F 13 260-279 97 97 97 164 164 164 T T T 14 280-299 98 98 98 166 166 166 F F F 15 300-319 100 100 100 169 169 169 F F F 16 320-339 100 100 100 169 169 169 T T T 17 340-359 100 100 100 169 169 169 T T T 18 360-379 96 96 96 163 163 163 F F F 19 380-399 87 87 87 152 152 152 T T T 20 400-419 100 100 100 169 169 169 F F F 21 420-439 100 100 100 169 169 169 F F F 22 440-459 98 98 98 166 166 166 F F F 23 460-479 97 97 97 164 164 164 F F F 24 480-499 100 100 100 169 169 169 F F F 25 500-519 100 100 100 169 169 169 F F F 26 520-539 97 97 97 164 164 164 F F F 27 540-559 94 94 94 160 160 160 F F F 28 560-579 100 100 100 168 168 168 T T T 29 580-599 96 96 96 163 163 163 T T T Max gen: 100, Max flush: 169 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 165 165 165 F F F 1 020-039 101 101 101 170 170 170 F F F 2 040-059 96 96 96 163 163 163 F F F 3 060-079 101 101 101 170 170 170 F F F 4 080-099 98 98 98 165 165 165 F F F 5 100-119 98 98 98 165 165 165 F F F 6 120-139 88 88 88 154 154 154 F F F 7 140-159 97 97 97 164 164 164 F F F 8 160-179 101 101 101 170 170 170 F F F 9 180-199 101 101 101 170 170 170 F F F 10 200-219 100 100 100 169 169 169 F F F 11 220-239 87 87 87 152 152 152 F F F 12 240-259 99 99 99 167 167 167 F F F 13 260-279 101 101 101 170 170 170 F F F 14 280-299 98 98 98 166 166 166 F F F 15 300-319 100 100 100 169 169 169 F F F 16 320-339 101 101 101 170 170 170 F F F 17 340-359 101 101 101 170 170 170 F F F 18 360-379 96 96 96 163 163 163 F F F 19 380-399 101 101 101 170 170 170 F F F 20 400-419 100 100 100 169 169 169 F F F 21 420-439 100 100 100 169 169 169 F F F 22 440-459 98 98 98 166 166 166 F F F 23 460-479 97 97 97 164 164 164 F F F 24 480-499 100 100 100 169 169 169 F F F 25 500-519 100 100 100 169 169 169 F F F 26 520-539 97 97 97 164 164 164 F F F 27 540-559 94 94 94 160 160 160 F F F 28 560-579 101 101 101 170 170 170 F F F 29 580-599 101 101 101 170 170 170 F F F Max gen: 101, Max flush: 170 Begin loop 51 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.820695669Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.821116855Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.821151033Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.821181712Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590} {"msg":"Crucible 6990753b-aa1e-4ce5-a010-5bcc6fd50782 has session id: 511732de-8210-4f5b-8a76-379a668a4dcc","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.82120687Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"Upstairs opts: Upstairs UUID: 6990753b-aa1e-4ce5-a010-5bcc6fd50782, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.821256467Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"Crucible stats registered with UUID: 6990753b-aa1e-4ce5-a010-5bcc6fd50782","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.821284456Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.856389118Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590} {"msg":"6990753b-aa1e-4ce5-a010-5bcc6fd50782 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.856419826Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.856437565Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"client":"0","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.856468903Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"client":"0","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.856486762Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"client":"1","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.856504141Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"client":"1","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.85652097Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"client":"2","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.856546019Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"client":"2","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.856576837Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.856609175Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"":"io task","client":"0","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.856643663Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"ds_connection connected from Ok(127.0.0.1:62241)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.856680601Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"":"io task","client":"0","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"ds_connection connected from Ok(127.0.0.1:51505)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.856817533Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"ds_connection connected from Ok(127.0.0.1:44306)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.856868451Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.857424069Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"client":"2","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.857458217Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"client":"2","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.857486476Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"client":"0","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.857555702Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"client":"0","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.85758448Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"client":"1","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.857628088Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"client":"1","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.858131089Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"client":"0","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.858160137Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"client":"2","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.858247743Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"client":"1","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.858733145Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"client":"2","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.858768743Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"client":"2","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.858795152Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.85882043Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"client":"1","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.858846689Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"client":"1","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.858887506Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.858911525Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"client":"0","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.858934044Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"client":"0","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"[0]R flush_numbers[0..12]: [165, 170, 163, 170, 165, 165, 154, 164, 170, 170, 169, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.858958102Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"":"downstairs","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"[0]R generation[0..12]: [98, 101, 96, 101, 98, 98, 88, 97, 101, 101, 100, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.858982731Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"":"downstairs","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.859013149Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"":"downstairs","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"[1]R flush_numbers[0..12]: [165, 170, 163, 170, 165, 165, 154, 164, 170, 170, 169, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.859037958Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"":"downstairs","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"[1]R generation[0..12]: [98, 101, 96, 101, 98, 98, 88, 97, 101, 101, 100, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.859060826Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"":"downstairs","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.859085715Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"":"downstairs","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"[2]R flush_numbers[0..12]: [165, 170, 163, 170, 165, 165, 154, 164, 170, 170, 169, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.859113123Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"":"downstairs","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"[2]R generation[0..12]: [98, 101, 96, 101, 98, 98, 88, 97, 101, 101, 100, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.859138602Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"":"downstairs","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.859163321Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"":"downstairs","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.859188799Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3"Wait for a query_work_queue command to finish before sending IO ,"pid":1590,"":"downstairs","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.859220237Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"":"downstairs","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"Next flush: 171","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.859247806Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"":"downstairs","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.859274214Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"":"downstairs","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.859300893Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.859325371Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"client":"0","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.85935273Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"client":"1","":"downstairs"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.859390868Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"client":"2","":"downstairs","session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.859413897Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.859426986Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"6990753b-aa1e-4ce5-a010-5bcc6fd50782 is now active with session: 511732de-8210-4f5b-8a76-379a668a4dcc","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.859440045Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.859451964Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590,"session_id":"511732de-8210-4f5b-8a76-379a668a4dcc"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:49.859464074Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 325 len 16384 data: 9 8 11 12 02/30 Write block 285 len 36864 data: 11 10 9 10 12 12 11 10 9 03/30 Read block 400 len 4096 04/30 Write block 174 len 20480 data: 6 6 8 6 6 05/30 Read block 578 len 16384 06/30 Read block 54 len 24576 07/30 Write block 113 len 4096 data: 15 08/30 Read block 392 len 4096 09/30 Read block 148 len 16384 10/30 Read block 426 len 40960 11/30 Flush 12/30 Write block 350 len 36864 data: 8 10 11 13 11 11 8 10 11 13/30 Write block 501 len 16384 data: 11 11 10 9 14/30 Read block 226 len 4096 15/30 Write block 531 len 28672 data: 9 10 11 10 9 9 10 16/30 Flush 17/30 Write block 78 len 28672 data: 9 11 9 10 10 8 8 18/30 Write block 148 len 28672 data: 4 4 4 7 7 7 6 19/30 Read block 574 len 8192 20/30 Read block 483 len 40960 21/30 Flush 22/30 Write block 74 len 32768 data: 10 8 10 9 10 12 10 11 23/30 Write block 119 len 36864 data: 9 9 10 9 7 8 9 6 7 24/30 Read block 29 len 28672 25/30 Write block 210 len 28672 data: 8 7 7 10 10 10 12 26/30 Write block 244 len 4096 data: 7 27/30 Write block 141 len 36864 data: 6 4 5 5 6 6 8 5 5 28/30 Read block 574 len 8192 29/30 Read block 412 len 4096 30/30 Read block 112 len 24576 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 8 Done Done Done false 23 Acked 1022 Write 9 Done Done Done false 24 Acked 1023 Read 7 Done Done Done false 25 Acked 1024 Write 7 Done Done Done false 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Write 9 Done Done Done false 28 Acked 1027 Read 2 Done Done Done false 29 Acked 1028 Read 1 Done Done Done false 30 Acked 1029 Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 9 9 8 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:31:49.884130827Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1590} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 165 165 165 F F F 1 020-039 101 101 101 170 170 170 F F F 2 040-059 96 96 96 163 163 163 F F F 3 060-079 102 102 102 173 173 173 T T T 4 080-099 102 102 102 173 173 173 T T T 5 100-119 102 102 102 171 171 171 T T T 6 120-139 88 88 88 154 154 154 T T T 7 140-159 102 102 102 173 173 173 T T T 8 160-179 102 102 102 171 171 171 F F F 9 180-199 101 101 101 170 170 170 F F F 10 200-219 100 100 100 169 169 169 T T T 11 220-239 87 87 87 152 152 152 F F F 12 240-259 99 99 99 167 167 167 T T T 13 260-279 101 101 101 170 170 170 F F F 14 280-299 102 102 102 171 171 171 F F F 15 300-319 100 100 100 169 169 169 F F F 16 320-339 102 102 102 171 171 171 F F F 17 340-359 102 102 102 172 172 172 F F F 18 360-379 96 96 96 163 163 163 F F F 19 380-399 101 101 101 170 170 170 F F F 20 400-419 100 100 100 169 169 169 F F F 21 420-439 100 100 100 169 169 169 F F F 22 440-459 98 98 98 166 166 166 F F F 23 460-479 97 97 97 164 164 164 F F F 24 480-499 100 100 100 169 169 169 F F F 25 500-519 102 102 102 172 172 172 F F F 26 520-539 102 102 102 172 172 172 F F F 27 540-559 94 94 94 160 160 160 F F F 28 560-579 101 101 101 170 170 170 F F F 29 580-599 101 101 101 170 170 170 F F F Max gen: 102, Max flush: 173 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 165 165 165 F F F 1 020-039 101 101 101 170 170 170 F F F 2 040-059 96 96 96 163 163 163 F F F 3 060-079 103 103 103 174 174 174 F F F 4 080-099 103 103 103 174 174 174 F F F 5 100-119 103 103 103 174 174 174 F F F 6 120-139 103 103 103 174 174 174 F F F 7 140-159 103 103 103 174 174 174 F F F 8 160-179 102 102 102 171 171 171 F F F 9 180-199 101 101 101 170 170 170 F F F 10 200-219 103 103 103 174 174 174 F F F 11 220-239 87 87 87 152 152 152 F F F 12 240-259 103 103 103 174 174 174 F F F 13 260-279 101 101 101 170 170 170 F F F 14 280-299 102 102 102 171 171 171 F F F 15 300-319 100 100 100 169 169 169 F F F 16 320-339 102 102 102 171 171 171 F F F 17 340-359 102 102 102 172 172 172 F F F 18 360-379 96 96 96 163 163 163 F F F 19 380-399 101 101 101 170 170 170 F F F 20 400-419 100 100 100 169 169 169 F F F 21 420-439 100 100 100 169 169 169 F F F 22 440-459 98 98 98 166 166 166 F F F 23 460-479 97 97 97 164 164 164 F F F 24 480-499 100 100 100 169 169 169 F F F 25 500-519 102 102 102 172 172 172 F F F 26 520-539 102 102 102 172 172 172 F F F 27 540-559 94 94 94 160 160 160 F F F 28 560-579 101 101 101 170 170 170 F F F 29 580-599 101 101 101 170 170 170 F F F Max gen: 103, Max flush: 174 Begin loop 52 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.293194432Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.293489225Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.293511104Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.293528193Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598} {"msg":"Crucible fa8fc878-47e0-4ea3-bd0b-eecf47a4fc39 has session id: c9f19fcf-8ad9-4734-ab44-ea483ac04731","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.293638517Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"Upstairs opts: Upstairs UUID: fa8fc878-47e0-4ea3-bd0b-eecf47a4fc39, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.293683365Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"Crucible stats registered with UUID: fa8fc878-47e0-4ea3-bd0b-eecf47a4fc39","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.293700164Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.325068537Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598} {"msg":"fa8fc878-47e0-4ea3-bd0b-eecf47a4fc39 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.325098275Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.325116914Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"client":"0","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.325143993Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"client":"0","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.325161312Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"client":"1","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.325177771Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"client":"1","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.32519836Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"client":"2","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.325220039Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"client":"2","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.325237138Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.325254457Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.325271225Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"ds_connection connected from Ok(127.0.0.1:55967)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.325334452Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"ds_connection connected from Ok(127.0.0.1:39991)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.325502452Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"ds_connection connected from Ok(127.0.0.1:41214)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.32554793Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.326197713Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"client":"2","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.326225562Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"client":"2","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.32624572Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"client":"1","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.326269729Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"client":"1","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.326347735Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"client":"0","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.326376583Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"client":"0","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.326877645Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"client":"2","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.326928042Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"client":"1","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.327174608Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"client":"0","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.327441863Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"client":"2","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.327478671Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"client":"2","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.327503009Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.327524638Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"client":"1","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.327549087Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"client":"1","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.327575665Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.327727177Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"client":"0","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.327755895Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"client":"0","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"[0]R flush_numbers[0..12]: [165, 170, 163, 174, 174, 174, 174, 174, 171, 170, 174, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.327772214Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"":"downstairs","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"[0]R generation[0..12]: [98, 101, 96, 103, 103, 103, 103, 103, 102, 101, 103, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.327786983Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"":"downstairs","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.327801262Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"":"downstairs","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"[1]R flush_numbers[0..12]: [165, 170, 163, 174, 174, 174, 174, 174, 171, 170, 174, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.327815421Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"":"downstairs","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"[1]R generation[0..12]: [98, 101, 96, 103, 103, 103, 103, 103, 102, 101, 103, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.327829271Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"":"downstairs","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.32784795Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"":"downstairs","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"[2]R flush_numbers[0..12]: [165, 170, 163, 174, 174, 174, 174, 174, 171, 170, 174, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.327862159Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"":"downstairs","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"[2]R generation[0..12]: [98, 101, 96, 103, 103, 103, 103, 103, 102, 101, 103, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.327875788Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"":"downstairs","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.327889727Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"":"downstairs","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.327902627Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"":"downstairs","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.327916036Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"":"downstairs","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"Next flush: 175","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.327928995Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"":"downstairs","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"Wait for a query_work_queue command to finish before sending IO All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.327945034Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"":"downstairs","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.327967083Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.327991322Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"client":"0","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.32801555Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"client":"1","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.328040049Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"client":"2","":"downstairs","session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.328064437Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.328087486Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 fa8fc878-47e0-4ea3-bd0b-eecf47a4fc39 is now active with session: c9f19fcf-8ad9-4734-ab44-ea483ac04731","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.328114284Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.328141363Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598,"session_id":"c9f19fcf-8ad9-4734-ab44-ea483ac04731"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:54.328164372Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 44 len 12288 data: 5 6 8 02/30 Read block 332 len 16384 03/30 Write block 383 len 8192 data: 8 7 04/30 Flush 05/30 Write block 246 len 16384 data: 7 9 7 8 06/30 Read block 285 len 32768 07/30 Write block 198 len 4096 data: 8 08/30 Flush 09/30 Write block 430 len 4096 data: 9 10/30 Flush 11/30 Write block 314 len 16384 data: 13 12 14 14 12/30 Read block 358 len 16384 13/30 Write block 559 len 40960 data: 6 7 5 6 5 3 5 4 3 6 14/30 Read block 557 len 16384 15/30 Flush 16/30 Write block 503 len 32768 data: 11 10 8 7 8 8 7 8 17/30 Read block 323 len 36864 18/30 Read block 437 len 20480 19/30 Write block 195 len 16384 data: 6 7 7 9 20/30 Write block 370 len 36864 data: 6 6 6 5 4 4 3 4 6 21/30 Read block 295 len 20480 22/30 Read block 212 len 16384 23/30 Write block 102 len 40960 data: 9 10 10 10 8 8 10 9 8 12 24/30 Write block 541 len 40960 data: 9 8 9 10 14 13 12 12 10 10 25/30 Write block 56 len 24576 data: 8 11 9 9 9 8 26/30 Read block 107 len 4096 27/30 Read block 274 len 40960 28/30 Write block 42 len 20480 data: 5 6 6 7 9 29/30 Read block 220 len 32768 30/30 Write block 195 len 8192 data: 7 8 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 16 Acked 1015 Write 8 Done Done Done false 17 Acked 1016 Read 9 Done Done Done false 18 Acked 1017 Read 5 Done Done Done false 19 Acked 1018 Write 4 Done Done Done false 20 Acked 1019 Write 9 Done Done Done false 21 Acked 1020 Read 5 Done Done Done false 22 Acked 1021 Read 4 Done Done Done false 23 Acked 1022 Write 10 Done Done Done false 24 Acked 1023 Write 10 Done Done Done false 25 Acked 1024 Write 6 Done Done Done false 26 Acked 1025 Read 1 Done Done Done false 27 Acked 1026 Read 10 Done Done Done false 28 Acked 1027 Write 5 Done Sent Sent false 29 Acked 1028 Read 8 Done Sent Sent false 30 Acked 1029 Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 3 3 7 Done 14 12 12 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:31:54.353789401Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1598} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 165 165 165 F F F 1 020-039 101 101 101 170 170 170 F F F 2 040-059 104 104 104 175 175 175 T T T 3 060-079 103 103 103 174 174 174 T T T 4 080-099 103 103 103 174 174 174 F F F 5 100-119 103 103 103 174 174 174 T T T 6 120-139 103 103 103 174 174 174 F F F 7 140-159 103 103 103 174 174 174 F F F 8 160-179 102 102 102 171 171 171 F F F 9 180-199 104 104 104 176 176 176 T T T 10 200-219 103 103 103 174 174 174 F F F 11 220-239 87 87 87 152 152 152 F F F 12 240-259 104 104 104 176 176 176 F F F 13 260-279 101 101 101 170 170 170 F F F 14 280-299 102 102 102 171 171 171 F F F 15 300-319 104 104 104 178 178 178 F F F 16 320-339 102 102 102 171 171 171 F F F 17 340-359 102 102 102 172 172 172 F F F 18 360-379 96 96 96 163 163 163 T T T 19 380-399 104 104 104 175 175 175 F F F 20 400-419 100 100 100 169 169 169 F F F 21 420-439 104 104 104 177 177 177 F F F 22 440-459 98 98 98 166 166 166 F F F 23 460-479 97 97 97 164 164 164 F F F 24 480-499 100 100 100 169 169 169 F F F 25 500-519 102 102 102 172 172 172 T T T 26 520-539 102 102 102 172 172 172 F F F 27 540-559 104 104 104 178 178 178 T T T 28 560-579 104 104 104 178 178 178 F F F 29 580-599 101 101 101 170 170 170 F F F Max gen: 104, Max flush: 178 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 165 165 165 F F F 1 020-039 101 101 101 170 170 170 F F F 2 040-059 105 105 105 179 179 179 F F F 3 060-079 105 105 105 179 179 179 F F F 4 080-099 103 103 103 174 174 174 F F F 5 100-119 105 105 105 179 179 179 F F F 6 120-139 103 103 103 174 174 174 F F F 7 140-159 103 103 103 174 174 174 F F F 8 160-179 102 102 102 171 171 171 F F F 9 180-199 105 105 105 179 179 179 F F F 10 200-219 103 103 103 174 174 174 F F F 11 220-239 87 87 87 152 152 152 F F F 12 240-259 104 104 104 176 176 176 F F F 13 260-279 101 101 101 170 170 170 F F F 14 280-299 102 102 102 171 171 171 F F F 15 300-319 104 104 104 178 178 178 F F F 16 320-339 102 102 102 171 171 171 F F F 17 340-359 102 102 102 172 172 172 F F F 18 360-379 105 105 105 179 179 179 F F F 19 380-399 104 104 104 175 175 175 F F F 20 400-419 100 100 100 169 169 169 F F F 21 420-439 104 104 104 177 177 177 F F F 22 440-459 98 98 98 166 166 166 F F F 23 460-479 97 97 97 164 164 164 F F F 24 480-499 100 100 100 169 169 169 F F F 25 500-519 105 105 105 179 179 179 F F F 26 520-539 102 102 102 172 172 172 F F F 27 540-559 105 105 105 179 179 179 F F F 28 560-579 104 104 104 178 178 178 F F F 29 580-599 101 101 101 170 170 170 F F F Max gen: 105, Max flush: 179 Begin loop 53 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.763948143Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.764266425Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.764299044Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.764327722Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606} {"msg":"Crucible 9020e3bd-9bdc-42e6-ab92-ab6bc72ce643 has session id: b4857ac8-c15d-4f77-8e83-1bc55cb04988","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.764451985Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"Upstairs opts: Upstairs UUID: 9020e3bd-9bdc-42e6-ab92-ab6bc72ce643, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.764564509Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"Crucible stats registered with UUID: 9020e3bd-9bdc-42e6-ab92-ab6bc72ce643","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.764588837Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.796118812Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606} {"msg":"9020e3bd-9bdc-42e6-ab92-ab6bc72ce643 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.79614927Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.796173668Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"client":"0","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.796197577Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"client":"0","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.796225755Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"client":"1","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.796258404Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"client":"1","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.796291092Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"client":"2","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.796353348Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"client":"2","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.796397266Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"":"io task","client":"2","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.796434494Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.796563356Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"":"io task","client":"1","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"ds_connection connected from Ok(127.0.0.1:38643)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.796608584Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"":"io task","client":"2","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"ds_connection connected from Ok(127.0.0.1:38764)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.796642362Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"":"io task","client":"1","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"ds_connection connected from Ok(127.0.0.1:63202)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.79667394Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.797177281Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"client":"2","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.79720546Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"client":"2","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.797322443Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"client":"1","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.797342882Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"client":"1","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.797363331Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"client":"0","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.79738952Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"client":"0","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.797896641Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"client":"2","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.797936969Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"client":"0","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.798054592Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"client":"1","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.798576402Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"client":"2","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.798604601Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"client":"2","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.79861911Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.798697155Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"client":"1","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.798729954Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"client":"1","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.798751672Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.798765582Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"client":"0","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.798779201Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"client":"0","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"[0]R flush_numbers[0..12]: [165, 170, 179, 179, 174, 179, 174, 174, 171, 179, 174, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.79879313Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"":"downstairs","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"[0]R generation[0..12]: [98, 101, 105, 105, 103, 105, 103, 103, 102, 105, 103, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.798806989Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"":"downstairs","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.798821179Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"":"downstairs","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"[1]R flush_numbers[0..12]: [165, 170, 179, 179, 174, 179, 174, 174, 171, 179, 174, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.798834798Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"":"downstairs","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"[1]R generation[0..12]: [98, 101, 105, 105, 103, 105, 103, 103, 102, 105, 103, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.798848557Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"":"downstairs","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.798862316Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"":"downstairs","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"[2]R flush_numbers[0..12]: [165, 170, 179, 179, 174, 179, 174, 174, 171, 179, 174, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.798875995Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"":"downstairs","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"[2]R generation[0..12]: [98, 101, 105, 105, 103, 105, 103, 103, 102, 105, 103, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.798889424Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"":"downstairs","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.798906134Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"":"downstairs","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.798928402Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"":"downstairs","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2024-09-26T14:31:58.798956101Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"":"downstairs","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"Next flush: 180","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.798980839Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"":"downstairs","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.798996238Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"":"downstairs","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.799009738Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.799021737Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"client":"0","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.799034826Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"client":"1","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.799047816Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"client":"2","":"downstairs","session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.799060925Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.799073154Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"9020e3bd-9bdc-42e6-ab92-ab6bc72ce643 is now active with session: b4857ac8-c15d-4f77-8e83-1bc55cb04988","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.799085904Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.799097583Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606,"session_id":"b4857ac8-c15d-4f77-8e83-1bc55cb04988"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:31:58.799109892Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 179 len 24576 data: 8 10 8 10 12 12 02/30 Read block 154 len 12288 03/30 Write block 297 len 16384 data: 6 10 9 8 04/30 Flush 05/30 Write block 371 len 40960 data: 7 7 6 5 5 4 5 7 5 6 06/30 Flush 07/30 Write block 565 len 32768 data: 6 5 4 7 8 8 10 11 08/30 Read block 149 len 20480 09/30 Read block 52 len 36864 10/30 Read block 112 len 36864 11/30 Read block 483 len 24576 12/30 Write block 241 len 4096 data: 7 13/30 Read block 85 len 28672 14/30 Read block 287 len 12288 15/30 Read block 62 len 36864 16/30 Read block 322 len 24576 17/30 Write block 268 len 20480 data: 11 10 8 7 7 18/30 Read block 487 len 20480 19/30 Write block 350 len 40960 data: 9 11 12 14 12 12 9 11 12 10 20/30 Write block 455 len 32768 data: 9 10 10 8 7 6 7 4 21/30 Read block 477 len 12288 22/30 Read block 79 len 36864 23/30 Read block 92 len 16384 24/30 Flush 25/30 Write block 27 len 28672 data: 8 9 5 6 6 6 6 26/30 Write block 510 len 20480 data: 9 7 8 6 6 27/30 Read block 144 len 40960 28/30 Write block 167 len 40960 data: 7 6 8 9 9 8 8 7 7 9 29/30 Write block 381 len 8192 data: 5 7 30/30 Write block 502 len 24576 data: 12 12 11 9 8 9 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 7 Done Done Done false 26 Acked 1025 Write 5 Done Done Done false 27 Acked 1026 Read 10 Done Done Done false 28 Acked 1027 Write 10 Sent Sent Sent false 29 Acked 1028 Write 2 Sent Sent Sent false 30 Acked 1029 Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:31:58.827235289Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1606} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 165 165 165 F F F 1 020-039 101 101 101 170 170 170 T T T 2 040-059 105 105 105 179 179 179 F F F 3 060-079 105 105 105 179 179 179 F F F 4 080-099 103 103 103 174 174 174 F F F 5 100-119 105 105 105 179 179 179 F F F 6 120-139 103 103 103 174 174 174 F F F 7 140-159 103 103 103 174 174 174 F F F 8 160-179 106 106 106 180 180 180 T T T 9 180-199 106 106 106 180 180 180 F F F 10 200-219 103 103 103 174 174 174 F F F 11 220-239 87 87 87 152 152 152 F F F 12 240-259 106 106 106 182 182 182 F F F 13 260-279 106 106 106 182 182 182 F F F 14 280-299 106 106 106 180 180 180 F F F 15 300-319 106 106 106 180 180 180 F F F 16 320-339 102 102 102 171 171 171 F F F 17 340-359 106 106 106 182 182 182 F F F 18 360-379 106 106 106 181 181 181 F F F 19 380-399 106 106 106 181 181 181 T T T 20 400-419 100 100 100 169 169 169 F F F 21 420-439 104 104 104 177 177 177 F F F 22 440-459 106 106 106 182 182 182 F F F 23 460-479 106 106 106 182 182 182 F F F 24 480-499 100 100 100 169 169 169 F F F 25 500-519 105 105 105 179 179 179 T T T 26 520-539 102 102 102 172 172 172 F F F 27 540-559 105 105 105 179 179 179 F F F 28 560-579 106 106 106 182 182 182 F F F 29 580-599 101 101 101 170 170 170 F F F Max gen: 106, Max flush: 182 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 165 165 165 F F F 1 020-039 107 107 107 183 183 183 F F F 2 040-059 105 105 105 179 179 179 F F F 3 060-079 105 105 105 179 179 179 F F F 4 080-099 103 103 103 174 174 174 F F F 5 100-119 105 105 105 179 179 179 F F F 6 120-139 103 103 103 174 174 174 F F F 7 140-159 103 103 103 174 174 174 F F F 8 160-179 107 107 107 183 183 183 F F F 9 180-199 106 106 106 180 180 180 F F F 10 200-219 103 103 103 174 174 174 F F F 11 220-239 87 87 87 152 152 152 F F F 12 240-259 106 106 106 182 182 182 F F F 13 260-279 106 106 106 182 182 182 F F F 14 280-299 106 106 106 180 180 180 F F F 15 300-319 106 106 106 180 180 180 F F F 16 320-339 102 102 102 171 171 171 F F F 17 340-359 106 106 106 182 182 182 F F F 18 360-379 106 106 106 181 181 181 F F F 19 380-399 107 107 107 183 183 183 F F F 20 400-419 100 100 100 169 169 169 F F F 21 420-439 104 104 104 177 177 177 F F F 22 440-459 106 106 106 182 182 182 F F F 23 460-479 106 106 106 182 182 182 F F F 24 480-499 100 100 100 169 169 169 F F F 25 500-519 107 107 107 183 183 183 F F F 26 520-539 102 102 102 172 172 172 F F F 27 540-559 105 105 105 179 179 179 F F F 28 560-579 106 106 106 182 182 182 F F F 29 580-599 101 101 101 170 170 170 F F F Max gen: 107, Max flush: 183 Begin loop 54 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.155851381Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.156279797Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.156308115Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.156334094Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614} {"msg":"Crucible 73e0fad7-f01c-4f77-81ad-a60b78beaf1b has session id: 11a12b8c-f00e-4757-bc7d-537e2222cd8f","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.156469016Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"Upstairs opts: Upstairs UUID: 73e0fad7-f01c-4f77-81ad-a60b78beaf1b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.156518184Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"Crucible stats registered with UUID: 73e0fad7-f01c-4f77-81ad-a60b78beaf1b","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.156546002Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.18747867Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614} {"msg":"73e0fad7-f01c-4f77-81ad-a60b78beaf1b active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.187510308Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.187525687Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"client":"0","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.187544816Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"client":"0","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.187558955Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"client":"1","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.187572915Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"client":"1","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.187586434Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"client":"2","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.187599763Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"client":"2","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.187613232Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.187627072Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"":"io task","client":"0","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.187640831Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"ds_connection connected from Ok(127.0.0.1:64066)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.187939054Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"":"io task","client":"0","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"ds_connection connected from Ok(127.0.0.1:58137)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.187982991Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"ds_connection connected from Ok(127.0.0.1:58299)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.18800422Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.188506712Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"client":"2","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.18853415Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"client":"2","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.188549989Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"client":"0","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.188568228Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"client":"0","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.188811335Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"client":"1","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.188836853Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"client":"1","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.189212842Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"client":"0","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.18941453Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"client":"2","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.189706454Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"client":"1","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.189873295Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"client":"0","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.189901423Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"client":"0","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.189916412Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.189940681Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"client":"2","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.18995486Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"client":"2","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.189968319Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.190207145Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"client":"1","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.190234234Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"client":"1","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"[0]R flush_numbers[0..12]: [165, 183, 179, 179, 174, 179, 174, 174, 183, 180, 174, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.190249813Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"":"downstairs","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"[0]R generation[0..12]: [98, 107, 105, 105, 103, 105, 103, 103, 107, 106, 103, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.190264302Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"":"downstairs","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.190278412Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"":"downstairs","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"[1]R flush_numbers[0..12]: [165, 183, 179, 179, 174, 179, 174, 174, 183, 180, 174, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.190292511Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"":"downstairs","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"[1]R generation[0..12]: [98, 107, 105, 105, 103, 105, 103, 103, 107, 106, 103, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.19030616Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"":"downstairs","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.190320629Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"":"downstairs","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"[2]R flush_numbers[0..12]: [165, 183, 179, 179, 174, 179, 174, 174, 183, 180, 174, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.190334378Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"":"downstairs","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"[2]R generation[0..12]: [98, 107, 105, 105, 103, 105, 103, 103, 107, 106, 103, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.190348037Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"":"downstairs","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.190361947Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"":"downstairs","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.190374876Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"":"downstairs","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.190388005Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"":"downstairs","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"Next flush: 184","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.190400765Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"":"downstairs","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.190413824Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.190433733Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.190446112Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"client":"0","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.190459321Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"client":"1","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.190472461Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"client":"2","":"downstairs","session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.19048576Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.190498289Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"73e0fad7-f01c-4f77-81ad-a60b78beaf1b is now active with session: 11a12b8c-f00e-4757-bc7d-537e2222cd8f","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.190511138Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.190523048Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614,"session_id":"11a12b8c-f00e-4757-bc7d-537e2222cd8f"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:03.190535257Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 144 len 4096 data: 6 02/30 Read block 103 len 8192 03/30 Read block 305 len 16384 04/30 Flush 05/30 Write block 24 len 32768 data: 9 7 8 9 10 6 7 7 06/30 Read block 100 len 32768 07/30 Read block 178 len 12288 08/30 Read block 254 len 40960 09/30 Write block 104 len 8192 data: 11 11 10/30 Read block 93 len 28672 11/30 Write block 441 len 32768 data: 8 7 8 9 10 11 11 9 12/30 Read block 322 len 8192 13/30 Write block 356 len 36864 data: 10 12 13 11 11 9 10 9 9 14/30 Read block 404 len 32768 15/30 Read block 71 len 40960 16/30 Read block 191 len 16384 17/30 Read block 393 len 28672 18/30 Write block 370 len 24576 data: 7 8 8 7 6 6 19/30 Read block 564 len 8192 20/30 Read block 131 len 24576 21/30 Write block 440 len 24576 data: 7 9 8 9 10 11 22/30 Flush 23/30 Write block 121 len 20480 data: 11 10 8 9 10 24/30 Write block 489 len 32768 data: 8 8 8 9 9 11 11 10 25/30 Write block 281 len 36864 data: 10 11 9 10 12 11 10 11 13 26/30 Write block 385 len 24576 data: 7 5 5 4 4 5 27/30 Write block 200 len 32768 data: 6 5 6 5 8 8 8 6 28/30 Read block 384 len 16384 29/30 Write block 209 len 36864 data: 8 9 8 8 11 11 11 13 14 30/30 Write block 246 len 12288 data: 8 10 8 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 5 Done Done Done false 24 Acked 1023 Write 8 Done Done Done false 25 Acked 1024 Write 9 Done Done Done false 26 Acked 1025 Write 6 Done Done Done false 27 Acked 1026 Write 8 Done Done Done false 28 Acked 1027 Read 4 Done Done Done false 29 Acked 1028 Write 9 Sent Sent Sent false 30 Acked 1029 Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:03.214090873Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1614} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 165 165 165 F F F 1 020-039 108 108 108 185 185 185 F F F 2 040-059 105 105 105 179 179 179 F F F 3 060-079 105 105 105 179 179 179 F F F 4 080-099 103 103 103 174 174 174 F F F 5 100-119 108 108 108 185 185 185 F F F 6 120-139 103 103 103 174 174 174 T T T 7 140-159 108 108 108 184 184 184 F F F 8 160-179 107 107 107 183 183 183 F F F 9 180-199 106 106 106 180 180 180 F F F 10 200-219 103 103 103 174 174 174 T T T 11 220-239 87 87 87 152 152 152 F F F 12 240-259 106 106 106 182 182 182 T T T 13 260-279 106 106 106 182 182 182 F F F 14 280-299 106 106 106 180 180 180 T T T 15 300-319 106 106 106 180 180 180 F F F 16 320-339 102 102 102 171 171 171 F F F 17 340-359 108 108 108 185 185 185 F F F 18 360-379 108 108 108 185 185 185 F F F 19 380-399 107 107 107 183 183 183 T T T 20 400-419 100 100 100 169 169 169 F F F 21 420-439 104 104 104 177 177 177 F F F 22 440-459 108 108 108 185 185 185 F F F 23 460-479 106 106 106 182 182 182 F F F 24 480-499 100 100 100 169 169 169 T T T 25 500-519 107 107 107 183 183 183 F F F 26 520-539 102 102 102 172 172 172 F F F 27 540-559 105 105 105 179 179 179 F F F 28 560-579 106 106 106 182 182 182 F F F 29 580-599 101 101 101 170 170 170 F F F Max gen: 108, Max flush: 185 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 165 165 165 F F F 1 020-039 108 108 108 185 185 185 F F F 2 040-059 105 105 105 179 179 179 F F F 3 060-079 105 105 105 179 179 179 F F F 4 080-099 103 103 103 174 174 174 F F F 5 100-119 108 108 108 185 185 185 F F F 6 120-139 109 109 109 186 186 186 F F F 7 140-159 108 108 108 184 184 184 F F F 8 160-179 107 107 107 183 183 183 F F F 9 180-199 106 106 106 180 180 180 F F F 10 200-219 109 109 109 186 186 186 F F F 11 220-239 87 87 87 152 152 152 F F F 12 240-259 109 109 109 186 186 186 F F F 13 260-279 106 106 106 182 182 182 F F F 14 280-299 109 109 109 186 186 186 F F F 15 300-319 106 106 106 180 180 180 F F F 16 320-339 102 102 102 171 171 171 F F F 17 340-359 108 108 108 185 185 185 F F F 18 360-379 108 108 108 185 185 185 F F F 19 380-399 109 109 109 186 186 186 F F F 20 400-419 100 100 100 169 169 169 F F F 21 420-439 104 104 104 177 177 177 F F F 22 440-459 108 108 108 185 185 185 F F F 23 460-479 106 106 106 182 182 182 F F F 24 480-499 109 109 109 186 186 186 F F F 25 500-519 107 107 107 183 183 183 F F F 26 520-539 102 102 102 172 172 172 F F F 27 540-559 105 105 105 179 179 179 F F F 28 560-579 106 106 106 182 182 182 F F F 29 580-599 101 101 101 170 170 170 F F F Max gen: 109, Max flush: 186 Begin loop 55 Downstairs to restart: 2 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.579480852Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.579732738Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.579751286Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.579766516Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622} {"msg":"Crucible baede55e-2f73-45db-8c9f-10f577ec0b2c has session id: c4573245-d81f-459e-95a6-d02469c8f560","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.580146544Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"Upstairs opts: Upstairs UUID: baede55e-2f73-45db-8c9f-10f577ec0b2c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.580195181Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"Crucible stats registered with UUID: baede55e-2f73-45db-8c9f-10f577ec0b2c","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.58021768Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.611510798Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622} {"msg":"baede55e-2f73-45db-8c9f-10f577ec0b2c active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.611556075Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.611581774Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"client":"0","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.611622101Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"client":"0","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.61165449Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"client":"1","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.611749504Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"client":"1","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.611841459Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"client":"2","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.611961042Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"client":"2","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.61200386Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.612042078Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.612079635Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"ds_connection connected from Ok(127.0.0.1:46813)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.612119713Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"ds_connection connected from Ok(127.0.0.1:42729)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.612160501Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"ds_connection connected from Ok(127.0.0.1:47622)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.612202119Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.612958836Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"client":"0","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.613015242Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"client":"0","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.613040821Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"client":"1","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.61305614Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"client":"1","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.613070319Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"client":"2","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.613087638Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"client":"2","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.613565911Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"client":"0","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.613744901Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"client":"1","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.613874914Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"client":"2","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.614241023Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"client":"0","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.614265382Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"client":"0","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.614279621Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.614380865Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"client":"1","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.614399844Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"client":"1","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.614413513Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.614483719Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"client":"2","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.614501848Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"client":"2","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"[0]R flush_numbers[0..12]: [165, 185, 179, 179, 174, 185, 186, 184, 183, 180, 186, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.614517448Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"":"downstairs","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"[0]R generation[0..12]: [98, 108, 105, 105, 103, 108, 109, 108, 107, 106, 109, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.614531407Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"":"downstairs","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.614545616Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"":"downstairs","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"[1]R flush_numbers[0..12]: [165, 185, 179, 179, 174, 185, 186, 184, 183, 180, 186, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.614562925Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"":"downstairs","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"[1]R generation[0..12]: [98, 108, 105, 105, 103, 108, 109, 108, 107, 106, 109, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.614577044Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"":"downstairs","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.614590963Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"":"downstairs","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"[2]R flush_numbers[0..12]: [165, 185, 179, 179, 174, 185, 186, 184, 183, 180, 186, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.614604843Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"":"downstairs","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"[2]R generation[0..12]: [98, 108, 105, 105, 103, 108, 109, 108, 107, 106, 109, 87]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.614618552Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"":"downstairs","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.614632591Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"":"downstairs","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.61464605Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"":"downstairs","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.614659539Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"":"downstairs","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"Next flush: 187","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.614672489Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"":"downstairs","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.614685368Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"":"downstairs","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.614698557Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.614710677Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"client":"0","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.614728756Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"client":"1","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.614742735Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"client":"2","":"downstairs","session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.614757904Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.614770363Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"baede55e-2f73-45db-8c9f-10f577ec0b2c is now active with session: c4573245-d81f-459e-95a6-d02469c8f560","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.614783252Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.614795322Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622,"session_id":"c4573245-d81f-459e-95a6-d02469c8f560"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:07.614807471Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 78 len 40960 data: 11 13 11 12 11 9 9 9 14 13 02/30 Read block 577 len 16384 03/30 Read block 543 len 28672 04/30 Read block 312 len 28672 05/30 Read block 176 len 32768 06/30 Write block 182 len 32768 data: 11 13 13 11 7 8 8 6 07/30 Write block 346 len 4096 data: 8 08/30 Read block 364 len 28672 09/30 Read block 110 len 28672 10/30 Read block 292 len 28672 11/30 Flush 12/30 Write block 500 len 28672 data: 10 12 13 13 12 10 9 13/30 Read block 145 len 40960 14/30 Read block 473 len 16384 15/30 Flush 16/30 Write block 258 len 32768 data: 5 7 8 10 10 10 9 9 17/30 Write block 302 len 32768 data: 9 13 13 15 12 12 12 10 18/30 Read block 568 len 16384 19/30 Write block 541 len 12288 data: 10 9 10 20/30 Read block 251 len 36864 21/30 Read block 398 len 28672 22/30 Read block 62 len 4096 23/30 Write block 139 len 24576 data: 5 6 7 5 6 7 24/30 Flush 25/30 Write block 320 len 8192 data: 11 9 26/30 Read block 81 len 40960 27/30 Write block 224 len 24576 data: 8 7 6 6 7 5 28/30 Read block 481 len 32768 29/30 Read block 126 len 32768 30/30 Write block 94 len 40960 data: 6 3 4 4 5 6 8 11 10 11 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 2 Done Done Done false 26 Acked 1025 Read 10 Done Done Done false 27 Acked 1026 Write 6 Done Done Done false 28 Acked 1027 Read 8 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:07.640908053Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1622} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 165 165 165 F F F 1 020-039 108 108 108 185 185 185 F F F 2 040-059 105 105 105 179 179 179 F F F 3 060-079 110 110 110 187 187 187 F F F 4 080-099 110 110 110 187 187 187 T T T 5 100-119 108 108 108 185 185 185 T T T 6 120-139 110 110 110 189 189 189 F F F 7 140-159 110 110 110 189 189 189 F F F 8 160-179 107 107 107 183 183 183 F F F 9 180-199 110 110 110 187 187 187 F F F 10 200-219 109 109 109 186 186 186 F F F 11 220-239 87 87 87 152 152 152 T T T 12 240-259 110 110 110 189 189 189 F F F 13 260-279 110 110 110 189 189 189 F F F 14 280-299 109 109 109 186 186 186 F F F 15 300-319 110 110 110 189 189 189 F F F 16 320-339 102 102 102 171 171 171 T T T 17 340-359 110 110 110 187 187 187 F F F 18 360-379 108 108 108 185 185 185 F F F 19 380-399 109 109 109 186 186 186 F F F 20 400-419 100 100 100 169 169 169 F F F 21 420-439 104 104 104 177 177 177 F F F 22 440-459 108 108 108 185 185 185 F F F 23 460-479 106 106 106 182 182 182 F F F 24 480-499 109 109 109 186 186 186 F F F 25 500-519 110 110 110 188 188 188 F F F 26 520-539 102 102 102 172 172 172 F F F 27 540-559 110 110 110 189 189 189 F F F 28 560-579 106 106 106 182 182 182 F F F 29 580-599 101 101 101 170 170 170 F F F Max gen: 110, Max flush: 189 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 165 165 165 F F F 1 020-039 108 108 108 185 185 185 F F F 2 040-059 105 105 105 179 179 179 F F F 3 060-079 110 110 110 187 187 187 F F F 4 080-099 111 111 111 190 190 190 F F F 5 100-119 111 111 111 190 190 190 F F F 6 120-139 110 110 110 189 189 189 F F F 7 140-159 110 110 110 189 189 189 F F F 8 160-179 107 107 107 183 183 183 F F F 9 180-199 110 110 110 187 187 187 F F F 10 200-219 109 109 109 186 186 186 F F F 11 220-239 111 111 111 190 190 190 F F F 12 240-259 110 110 110 189 189 189 F F F 13 260-279 110 110 110 189 189 189 F F F 14 280-299 109 109 109 186 186 186 F F F 15 300-319 110 110 110 189 189 189 F F F 16 320-339 111 111 111 190 190 190 F F F 17 340-359 110 110 110 187 187 187 F F F 18 360-379 108 108 108 185 185 185 F F F 19 380-399 109 109 109 186 186 186 F F F 20 400-419 100 100 100 169 169 169 F F F 21 420-439 104 104 104 177 177 177 F F F 22 440-459 108 108 108 185 185 185 F F F 23 460-479 106 106 106 182 182 182 F F F 24 480-499 109 109 109 186 186 186 F F F 25 500-519 110 110 110 188 188 188 F F F 26 520-539 102 102 102 172 172 172 F F F 27 540-559 110 110 110 189 189 189 F F F 28 560-579 106 106 106 182 182 182 F F F 29 580-599 101 101 101 170 170 170 F F F Max gen: 111, Max flush: 190 Begin loop 56 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.009520089Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.009766135Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.009793894Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.009820212Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630} {"msg":"Crucible 179b6930-5dc1-4606-bbdf-be47590e45de has session id: c4d320c6-563e-49d4-b84f-d71feb0357f1","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.009918747Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"Upstairs opts: Upstairs UUID: 179b6930-5dc1-4606-bbdf-be47590e45de, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.009963164Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"Crucible stats registered with UUID: 179b6930-5dc1-4606-bbdf-be47590e45de","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.009986693Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.0411123Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630} {"msg":"179b6930-5dc1-4606-bbdf-be47590e45de active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.041148078Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.041165257Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"client":"0","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.041190026Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"client":"0","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.041207925Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"client":"1","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.041225124Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"client":"1","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.041241773Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"client":"2","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.041315359Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"client":"2","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.041357326Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.041386425Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.041410403Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"ds_connection connected from Ok(127.0.0.1:50324)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.041434892Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"ds_connection connected from Ok(127.0.0.1:37306)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.041461811Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"ds_connection connected from Ok(127.0.0.1:47249)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.041613902Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.042205638Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"client":"1","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.042238496Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"client":"1","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.042258555Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"client":"2","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.042281664Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"client":"2","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.0423512Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"client":"0","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.042369459Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"client":"0","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.042842952Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"client":"1","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.042973725Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"client":"0","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.042995494Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"client":"2","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.043377742Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"client":"1","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.0434139Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"client":"1","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.043439748Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.043625018Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"client":"0","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.043659246Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"client":"0","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.043682265Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.043703253Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"client":"2","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.043727582Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"client":"2","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"[0]R flush_numbers[0..12]: [165, 185, 179, 187, 190, 190, 189, 189, 183, 187, 186, 190]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.043752961Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"":"downstairs","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"[0]R generation[0..12]: [98, 108, 105, 110, 111, 111, 110, 110, 107, 110, 109, 111]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.043776809Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"":"downstairs","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.043800748Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"":"downstairs","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"[1]R flush_numbers[0..12]: [165, 185, 179, 187, 190, 190, 189, 189, 183, 187, 186, 190]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.043825057Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"":"downstairs","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"[1]R generation[0..12]: [98, 108, 105, 110, 111, 111, 110, 110, 107, 110, 109, 111]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.043851545Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"":"downstairs","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.043877804Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"":"downstairs","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"[2]R flush_numbers[0..12]: [165, 185, 179, 187, 190, 190, 189, 189, 183, 187, 186, 190]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.043902472Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"":"downstairs","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"[2]R generation[0..12]: [98, 108, 105, 110, 111, 111, 110, 110, 107, 110, 109, 111]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-26T14:32:12.04393492Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"":"downstairs","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.043965969Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"":"downstairs","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.043993367Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"":"downstairs","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.044020426Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"":"downstairs","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"Next flush: 191","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.044047554Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"":"downstairs","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.044074272Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"":"downstairs","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-09-26T14:32:12.044100731Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.044131979Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"client":"0","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.044154108Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"client":"1","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.044168107Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"client":"2","":"downstairs","session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.044181486Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.044193836Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"179b6930-5dc1-4606-bbdf-be47590e45de is now active with session: c4d320c6-563e-49d4-b84f-d71feb0357f1","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.044206825Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.044218974Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:12.044231364Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 545 len 24576 data: 15 14 13 13 11 11 02/30 Write block 43 len 4096 data: 7 03/30 Write block 360 len 16384 data: 12 10 11 10 04/30 Read block 276 len 24576 05/30 Write block 571 len 28672 data: 11 12 12 12 12 13 15 06/30 Read block 245 len 8192 07/30 Read block 321 len 12288 08/30 Read block 46 len 32768 09/30 Write block 10 len 24576 data: 7 8 8 8 7 8 10/30 Write block 355 len 12288 data: 13 11 13 11/30 Read block 304 len 36864 12/30 Read block 87 len 24576 13/30 Write block 103 len 24576 data: 12 12 12 9 9 11 14/30 Write block 249 len 40960 data: 9 6 5 6 5 7 8 7 4 6 15/30 Read block 230 len 12288 16/30 Flush 17/30 Write block 200 len 24576 data: 7 6 7 6 9 9 18/30 Flush 19/30 Write block 264 len 24576 data: 10 10 10 11 12 11 20/30 Read block 172 len 4096 21/30 Read block 185 len 28672 22/30 Write block 254 len 28672 data: 8 9 8 5 7 8 9 23/30 Write block 567 len 20480 data: 5 8 9 9 12 24/30 Read block 576 len 32768 25/30 Write block 495 len 16384 data: 12 11 11 11 26/30 Flush 27/30 Write block 194 len 8192 data: 6 8 28/30 Read block 395 len 36864 29/30 Write block 361 len 32768 data: 11 12 11 10 7 5 5 4 30/30 Write block 478 len 8192 data: 8 10 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 2 Done Done Done false 28 Acked 1027 Read 9 Done Done Done false 29 Acked 1028 Write 8 Sent Sent Sent false 30 Acked 1029 Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.068600143Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630} Wrote out file "/tmp/test_repair_verify.data" {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.06937543Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069410798Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069437496Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069461965Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069484753Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069506882Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069528661Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069555549Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069571739Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069584608Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069597037Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069609237Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069621336Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069633505Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069645784Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069657934Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069670043Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069682312Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069694432Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069706481Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.06971868Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069731079Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069743749Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069756068Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069770167Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069782717Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069795206Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069807585Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069820005Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069832414Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069844713Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069857062Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069869422Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069881561Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.06989393Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.06990623Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069918279Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069930598Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069942798Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069954967Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069966936Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069981045Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.069993465Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070013754Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070032273Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070044902Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070057121Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.07006914Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.0700814Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070093719Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070105808Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070117898Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070129927Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070141876Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070153895Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070165955Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070177944Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070189864Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070204083Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070216392Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070229291Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070241401Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.07025342Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070265629Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070277608Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070289768Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070302217Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070314626Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070326666Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070338805Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070350884Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070362824Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070374993Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070387052Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070398952Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070413071Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.07042534Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070437419Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070449429Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070461498Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070473608Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070485647Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070497666Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070509686Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070521685Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070533834Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070545853Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070557773Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070569812Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070581841Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070593761Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.07060584Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070619719Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070632208Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070644508Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070656937Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070669276Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070681596Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070693925Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070706204Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070718494Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070730803Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070743062Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070755481Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070767891Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.07078019Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.07079231Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070804469Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070816568Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070830417Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070842796Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070854866Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070866785Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070878925Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070891044Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070903133Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070915303Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070927362Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070939511Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070951581Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.07096361Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070975829Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:12.070987968Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1630,"session_id":"c4d320c6-563e-49d4-b84f-d71feb0357f1"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 191 191 191 F F F 1 020-039 108 108 108 185 185 185 F F F 2 040-059 112 112 112 191 191 191 F F F 3 060-079 110 110 110 187 187 187 F F F 4 080-099 111 111 111 190 190 190 F F F 5 100-119 112 112 112 191 191 191 F F F 6 120-139 110 110 110 189 189 189 F F F 7 140-159 110 110 110 189 189 189 F F F 8 160-179 107 107 107 183 183 183 F F F 9 180-199 110 110 110 187 187 187 T T T 10 200-219 112 112 112 192 192 192 F F F 11 220-239 111 111 111 190 190 190 F F F 12 240-259 112 112 112 193 193 193 F F F 13 260-279 112 112 112 193 193 193 F F F 14 280-299 109 109 109 186 186 186 F F F 15 300-319 110 110 110 189 189 189 F F F 16 320-339 111 111 111 190 190 190 F F F 17 340-359 112 112 112 191 191 191 F F F 18 360-379 112 112 112 191 191 191 T T T 19 380-399 109 109 109 186 186 186 F F F 20 400-419 100 100 100 169 169 169 F F F 21 420-439 104 104 104 177 177 177 F F F 22 440-459 108 108 108 185 185 185 F F F 23 460-479 106 106 106 182 182 182 T T T 24 480-499 112 112 112 193 193 193 F F F 25 500-519 110 110 110 188 188 188 F F F 26 520-539 102 102 102 172 172 172 F F F 27 540-559 112 112 112 191 191 191 F F F 28 560-579 112 112 112 193 193 193 F F F 29 580-599 101 101 101 170 170 170 F F F Max gen: 112, Max flush: 193 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 191 191 191 F F F 1 020-039 108 108 108 185 185 185 F F F 2 040-059 112 112 112 191 191 191 F F F 3 060-079 110 110 110 187 187 187 F F F 4 080-099 111 111 111 190 190 190 F F F 5 100-119 112 112 112 191 191 191 F F F 6 120-139 110 110 110 189 189 189 F F F 7 140-159 110 110 110 189 189 189 F F F 8 160-179 107 107 107 183 183 183 F F F 9 180-199 113 113 113 194 194 194 F F F 10 200-219 112 112 112 192 192 192 F F F 11 220-239 111 111 111 190 190 190 F F F 12 240-259 112 112 112 193 193 193 F F F 13 260-279 112 112 112 193 193 193 F F F 14 280-299 109 109 109 186 186 186 F F F 15 300-319 110 110 110 189 189 189 F F F 16 320-339 111 111 111 190 190 190 F F F 17 340-359 112 112 112 191 191 191 F F F 18 360-379 113 113 113 194 194 194 F F F 19 380-399 109 109 109 186 186 186 F F F 20 400-419 100 100 100 169 169 169 F F F 21 420-439 104 104 104 177 177 177 F F F 22 440-459 108 108 108 185 185 185 F F F 23 460-479 113 113 113 194 194 194 F F F 24 480-499 112 112 112 193 193 193 F F F 25 500-519 110 110 110 188 188 188 F F F 26 520-539 102 102 102 172 172 172 F F F 27 540-559 112 112 112 191 191 191 F F F 28 560-579 112 112 112 193 193 193 F F F 29 580-599 101 101 101 170 170 170 F F F Max gen: 113, Max flush: 194 Begin loop 57 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.456530616Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.456813851Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.456848858Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.456877277Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638} {"msg":"Crucible c6df1522-04fe-4edf-b78a-5cc5ee59dcfa has session id: cf6f3dfa-740b-4c92-bc59-dc005ee73634","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.457027479Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"Upstairs opts: Upstairs UUID: c6df1522-04fe-4edf-b78a-5cc5ee59dcfa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.457073096Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"Crucible stats registered with UUID: c6df1522-04fe-4edf-b78a-5cc5ee59dcfa","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.457090645Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.488182314Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638} {"msg":"c6df1522-04fe-4edf-b78a-5cc5ee59dcfa active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.488215542Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.488236801Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"client":"0","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.488280109Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"client":"0","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.488314316Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"client":"1","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.488348805Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"client":"1","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.488380673Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"client":"2","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.488412101Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"client":"2","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.488445349Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.488481557Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.488512155Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"ds_connection connected from Ok(127.0.0.1:53727)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.488545143Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"ds_connection connected from Ok(127.0.0.1:51936)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.488593671Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"ds_connection connected from Ok(127.0.0.1:53786)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.488743402Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.489288341Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"client":"2","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.489326109Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"client":"2","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.489343278Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"client":"1","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.489379586Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"client":"1","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.489394385Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"client":"0","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.489408234Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"client":"0","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.489926165Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"client":"1","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.490105355Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"client":"0","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.490134033Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"client":"2","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.490493783Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"client":"1","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.490521482Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"client":"1","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.490536491Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.490695052Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"client":"0","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.490723Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"client":"0","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.490745139Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.490758418Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"client":"2","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.490772307Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"client":"2","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"[0]R flush_numbers[0..12]: [191, 185, 191, 187, 190, 191, 189, 189, 183, 194, 192, 190]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.490786347Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"":"downstairs","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"[0]R generation[0..12]: [112, 108, 112, 110, 111, 112, 110, 110, 107, 113, 112, 111]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.490800256Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"":"downstairs","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.490814285Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"":"downstairs","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"[1]R flush_numbers[0..12]: [191, 185, 191, 187, 190, 191, 189, 189, 183, 194, 192, 190]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.490828064Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"":"downstairs","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"[1]R generation[0..12]: [112, 108, 112, 110, 111, 112, 110, 110, 107, 113, 112, 111]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.490841783Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"":"downstairs","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.490855583Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"":"downstairs","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"[2]R flush_numbers[0..12]: [191, 185, 191, 187, 190, 191, 189, 189, 183, 194, 192, 190]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.490869212Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"":"downstairs","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"[2]R generation[0..12]: [112, 108, 112, 110, 111, 112, 110, 110, 107, 113, 112, 111]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.490882741Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"":"downstairs","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.49090053Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"":"downstairs","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.490914079Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"":"downstairs","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.490927148Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"":"downstairs","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"Next flush: 195","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.490940098Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"":"downstairs","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg"Wait for a query_work_queue command to finish before sending IO :"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.490956817Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"":"downstairs","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.490970186Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.490982216Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"client":"0","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.490995335Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"client":"1","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.491008474Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"client":"2","":"downstairs","session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.491021493Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.491033673Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"c6df1522-04fe-4edf-b78a-5cc5ee59dcfa is now active with session: cf6f3dfa-740b-4c92-bc59-dc005ee73634","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.491046502Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.491058591Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638,"session_id":"cf6f3dfa-740b-4c92-bc59-dc005ee73634"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:16.491070791Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 553 len 4096 data: 5 02/30 Read block 577 len 20480 03/30 Write block 413 len 36864 data: 6 7 7 9 8 8 7 10 10 04/30 Write block 125 len 40960 data: 11 7 8 7 7 7 7 8 8 7 05/30 Read block 547 len 12288 06/30 Write block 504 len 20480 data: 13 11 10 10 9 07/30 Write block 9 len 8192 data: 6 8 08/30 Write block 26 len 12288 data: 9 10 11 09/30 Read block 409 len 32768 10/30 Flush 11/30 Write block 456 len 16384 data: 11 11 9 8 12/30 Write block 386 len 32768 data: 6 6 5 5 6 6 6 5 13/30 Read block 116 len 40960 14/30 Write block 48 len 36864 data: 11 10 12 10 11 11 7 8 9 15/30 Read block 249 len 4096 16/30 Read block 188 len 40960 17/30 Write block 572 len 12288 data: 13 13 13 18/30 Write block 585 len 36864 data: 9 9 9 8 8 7 5 5 5 19/30 Write block 526 len 12288 data: 10 9 9 20/30 Read block 2 len 20480 21/30 Write block 119 len 28672 data: 10 10 12 11 9 10 12 22/30 Write block 121 len 24576 data: 13 12 10 11 13 8 23/30 Read block 28 len 8192 24/30 Write block 374 len 36864 data: 7 7 5 6 8 6 7 6 8 25/30 Write block 583 len 20480 data: 9 8 10 10 10 26/30 Read block 488 len 32768 27/30 Write block 551 len 12288 data: 9 6 6 28/30 Read block 81 len 4096 29/30 Read block 61 len 12288 30/30 Read block 120 len 28672 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 11 Acked 1010 Write 4 Done Done Done false 12 Acked 1011 Write 8 Done Done Done false 13 Acked 1012 Read 10 Done Done Done false 14 Acked 1013 Write 9 Done Done Done false 15 Acked 1014 Read 1 Done Done Done false 16 Acked 1015 Read 10 Done Done Done false 17 Acked 1016 Write 3 Done Done Done false 18 Acked 1017 Write 9 Done Done Done false 19 Acked 1018 Write 3 Done Done Done false 20 Acked 1019 Read 5 Done Done Done false 21 Acked 1020 Write 7 Done Done Done false 22 Acked 1021 Write 6 Done Done Done false 23 Acked 1022 Read 2 Done Done Done false 24 Acked 1023 Write 9 Done Done Done false 25 Acked 1024 Write 5 Done Done Done false 26 Acked 1025 Read 8 Done Done Done false 27 Acked 1026 Write 3 Done Done Done false 28 Acked 1027 Read 1 Done Done Done false 29 Acked 1028 Read 3 Done Done Done false 30 Acked 1029 Read 7 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 19 19 20 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1009 1009 1009 Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:16.513790194Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1638} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 195 195 195 F F F 1 020-039 114 114 114 195 195 195 F F F 2 040-059 112 112 112 191 191 191 T T T 3 060-079 110 110 110 187 187 187 F F F 4 080-099 111 111 111 190 190 190 F F F 5 100-119 112 112 112 191 191 191 T T T 6 120-139 114 114 114 195 195 195 T T T 7 140-159 110 110 110 189 189 189 F F F 8 160-179 107 107 107 183 183 183 F F F 9 180-199 113 113 113 194 194 194 F F F 10 200-219 112 112 112 192 192 192 F F F 11 220-239 111 111 111 190 190 190 F F F 12 240-259 112 112 112 193 193 193 F F F 13 260-279 112 112 112 193 193 193 F F F 14 280-299 109 109 109 186 186 186 F F F 15 300-319 110 110 110 189 189 189 F F F 16 320-339 111 111 111 190 190 190 F F F 17 340-359 112 112 112 191 191 191 F F F 18 360-379 113 113 113 194 194 194 T T T 19 380-399 109 109 109 186 186 186 T T T 20 400-419 114 114 114 195 195 195 F F F 21 420-439 114 114 114 195 195 195 F F F 22 440-459 108 108 108 185 185 185 T T T 23 460-479 113 113 113 194 194 194 F F F 24 480-499 112 112 112 193 193 193 F F F 25 500-519 114 114 114 195 195 195 F F F 26 520-539 102 102 102 172 172 172 T T T 27 540-559 114 114 114 195 195 195 T T T 28 560-579 112 112 112 193 193 193 T T T 29 580-599 101 101 101 170 170 170 T T T Max gen: 114, Max flush: 195 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 195 195 195 F F F 1 020-039 114 114 114 195 195 195 F F F 2 040-059 115 115 115 196 196 196 F F F 3 060-079 110 110 110 187 187 187 F F F 4 080-099 111 111 111 190 190 190 F F F 5 100-119 115 115 115 196 196 196 F F F 6 120-139 115 115 115 196 196 196 F F F 7 140-159 110 110 110 189 189 189 F F F 8 160-179 107 107 107 183 183 183 F F F 9 180-199 113 113 113 194 194 194 F F F 10 200-219 112 112 112 192 192 192 F F F 11 220-239 111 111 111 190 190 190 F F F 12 240-259 112 112 112 193 193 193 F F F 13 260-279 112 112 112 193 193 193 F F F 14 280-299 109 109 109 186 186 186 F F F 15 300-319 110 110 110 189 189 189 F F F 16 320-339 111 111 111 190 190 190 F F F 17 340-359 112 112 112 191 191 191 F F F 18 360-379 115 115 115 196 196 196 F F F 19 380-399 115 115 115 196 196 196 F F F 20 400-419 114 114 114 195 195 195 F F F 21 420-439 114 114 114 195 195 195 F F F 22 440-459 115 115 115 196 196 196 F F F 23 460-479 113 113 113 194 194 194 F F F 24 480-499 112 112 112 193 193 193 F F F 25 500-519 114 114 114 195 195 195 F F F 26 520-539 115 115 115 196 196 196 F F F 27 540-559 115 115 115 196 196 196 F F F 28 560-579 115 115 115 196 196 196 F F F 29 580-599 115 115 115 196 196 196 F F F Max gen: 115, Max flush: 196 Begin loop 58 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.958941515Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.959296315Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.959325274Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.959354092Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646} {"msg":"Crucible ff699bb9-a69c-47ac-9a94-c1c5856ec7c2 has session id: 24f122b3-3ce2-4428-a108-69a40c3f4538","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.9593928Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"Upstairs opts: Upstairs UUID: ff699bb9-a69c-47ac-9a94-c1c5856ec7c2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.959468355Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"Crucible stats registered with UUID: ff699bb9-a69c-47ac-9a94-c1c5856ec7c2","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.959495254Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.990662679Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646} {"msg":"ff699bb9-a69c-47ac-9a94-c1c5856ec7c2 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.990693117Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.990711036Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"client":"0","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.990739044Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"client":"0","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.990756493Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"client":"1","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.990778112Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"client":"1","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.99080784Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"client":"2","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.990838759Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"client":"2","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.990869077Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.990900395Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.990933153Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"ds_connection connected from Ok(127.0.0.1:59394)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.99099842Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"ds_connection connected from Ok(127.0.0.1:52027)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.991241546Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"ds_connection connected from Ok(127.0.0.1:58228)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.991285384Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.991843502Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"client":"2","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.991889529Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"client":"2","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.991918737Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"client":"0","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.991967845Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"client":"0","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.992013912Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"client":"1","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.992030751Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"client":"1","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.99257593Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"client":"2","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.992654206Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"client":"0","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.992742631Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"client":"1","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.993128279Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"client":"2","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.993155507Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"client":"2","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.993171017Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.993200805Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"client":"0","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.993215464Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"client":"0","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.993229053Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.99329519Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"client":"1","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.993314438Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"client":"1","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"[0]R flush_numbers[0..12]: [195, 195, 196, 187, 190, 196, 196, 189, 183, 194, 192, 190]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.993328658Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"":"downstairs","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"[0]R generation[0..12]: [114, 114, 115, 110, 111, 115, 115, 110, 107, 113, 112, 111]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.993342527Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"":"downstairs","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.993356506Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"":"downstairs","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"[1]R flush_numbers[0..12]: [195, 195, 196, 187, 190, 196, 196, 189, 183, 194, 192, 190]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.993370215Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"":"downstairs","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"[1]R generation[0..12]: [114, 114, 115, 110, 111, 115, 115, 110, 107, 113, 112, 111]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.993383925Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"":"downstairs","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.993397724Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"":"downstairs","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"[2]R flush_numbers[0..12]: [195, 195, 196, 187, 190, 196, 196, 189, 183, 194, 192, 190]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.993411593Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"":"downstairs","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"[2]R generation[0..12]: [114, 114, 115, 110, 111, 115, 115, 110, 107, 113, 112, 111]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.993425442Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"":"downstairs","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.993439651Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"":"downstairs","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.993455501Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"":"downstairs","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.99346895Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"":"downstairs","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"Next flush: 197","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.993482129Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"":"downstairs","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.993495028Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"":"downstairs","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.993508007Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"Wait for a query_work_queue command to finish before sending IO } {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.993523267Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"client":"0","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.993536316Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"client":"1","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.993549445Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"client":"2","":"downstairs","session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.993562615Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.993574724Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"ff699bb9-a69c-47ac-9a94-c1c5856ec7c2 is now active with session: 24f122b3-3ce2-4428-a108-69a40c3f4538","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.993587493Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.993599402Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646,"session_id":"24f122b3-3ce2-4428-a108-69a40c3f4538"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:20.993611512Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 571 len 16384 data: 13 14 14 14 02/30 Read block 584 len 16384 03/30 Read block 412 len 40960 04/30 Write block 536 len 16384 data: 10 11 10 10 05/30 Write block 298 len 32768 data: 11 10 9 5 10 14 14 16 06/30 Read block 59 len 12288 07/30 Flush 08/30 Flush 09/30 Write block 368 len 4096 data: 5 10/30 Read block 531 len 4096 11/30 Read block 149 len 12288 12/30 Read block 459 len 40960 13/30 Write block 66 len 28672 data: 11 10 10 7 7 9 10 14/30 Read block 496 len 8192 15/30 Read block 127 len 40960 16/30 Read block 341 len 28672 17/30 Read block 371 len 28672 18/30 Read block 13 len 20480 19/30 Read block 509 len 24576 20/30 Write block 282 len 40960 data: 12 10 11 13 12 11 12 14 13 12 21/30 Write block 274 len 20480 data: 9 12 13 13 14 22/30 Read block 70 len 40960 23/30 Write block 121 len 16384 data: 14 13 11 12 24/30 Read block 3 len 12288 25/30 Read block 400 len 8192 26/30 Write block 154 len 16384 data: 7 9 13 12 27/30 Read block 1 len 12288 28/30 Read block 565 len 20480 29/30 Write block 78 len 20480 data: 12 14 12 13 12 30/30 Read block 337 len 8192 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 9 Acked 1008 Write 1 Done Done Done false 10 Acked 1009 Read 1 Done Done Done false 11 Acked 1010 Read 3 Done Done Done false 12 Acked 1011 Read 10 Done Done Done false 13 Acked 1012 Write 7 Done Done Done false 14 Acked 1013 Read 2 Done Done Done false 15 Acked 1014 Read 10 Done Done Done false 16 Acked 1015 Read 7 Done Done Done false 17 Acked 1016 Read 7 Done Done Done false 18 Acked 1017 Read 5 Done Done Done false 19 Acked 1018 Read 6 Done Done Done false 20 Acked 1019 Write 10 Done Done Done false 21 Acked 1020 Write 5 Done Done Done false 22 Acked 1021 Read 10 Done Done Done false 23 Acked 1022 Write 4 Done Done Done false 24 Acked 1023 Read 3 Done Done Done false 25 Acked 1024 Read 2 Done Done Done false 26 Acked 1025 Write 4 Done Done Done false 27 Acked 1026 Read 3 Done Done Done false 28 Acked 1027 Read 5 Done Done Done false 29 Acked 1028 Write 5 Done Done Done false 30 Acked 1029 Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1007 1007 1007 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:21.017248693Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1646} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 195 195 195 F F F 1 020-039 114 114 114 195 195 195 F F F 2 040-059 115 115 115 196 196 196 F F F 3 060-079 110 110 110 187 187 187 T T T 4 080-099 111 111 111 190 190 190 T T T 5 100-119 115 115 115 196 196 196 F F F 6 120-139 115 115 115 196 196 196 T T T 7 140-159 110 110 110 189 189 189 T T T 8 160-179 107 107 107 183 183 183 F F F 9 180-199 113 113 113 194 194 194 F F F 10 200-219 112 112 112 192 192 192 F F F 11 220-239 111 111 111 190 190 190 F F F 12 240-259 112 112 112 193 193 193 F F F 13 260-279 112 112 112 193 193 193 T T T 14 280-299 116 116 116 197 197 197 T T T 15 300-319 116 116 116 197 197 197 F F F 16 320-339 111 111 111 190 190 190 F F F 17 340-359 112 112 112 191 191 191 F F F 18 360-379 115 115 115 196 196 196 T T T 19 380-399 115 115 115 196 196 196 F F F 20 400-419 114 114 114 195 195 195 F F F 21 420-439 114 114 114 195 195 195 F F F 22 440-459 115 115 115 196 196 196 F F F 23 460-479 113 113 113 194 194 194 F F F 24 480-499 112 112 112 193 193 193 F F F 25 500-519 114 114 114 195 195 195 F F F 26 520-539 116 116 116 197 197 197 F F F 27 540-559 115 115 115 196 196 196 F F F 28 560-579 116 116 116 197 197 197 F F F 29 580-599 115 115 115 196 196 196 F F F Max gen: 116, Max flush: 197 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 195 195 195 F F F 1 020-039 114 114 114 195 195 195 F F F 2 040-059 115 115 115 196 196 196 F F F 3 060-079 117 117 117 198 198 198 F F F 4 080-099 117 117 117 198 198 198 F F F 5 100-119 115 115 115 196 196 196 F F F 6 120-139 117 117 117 198 198 198 F F F 7 140-159 117 117 117 198 198 198 F F F 8 160-179 107 107 107 183 183 183 F F F 9 180-199 113 113 113 194 194 194 F F F 10 200-219 112 112 112 192 192 192 F F F 11 220-239 111 111 111 190 190 190 F F F 12 240-259 112 112 112 193 193 193 F F F 13 260-279 117 117 117 198 198 198 F F F 14 280-299 117 117 117 198 198 198 F F F 15 300-319 116 116 116 197 197 197 F F F 16 320-339 111 111 111 190 190 190 F F F 17 340-359 112 112 112 191 191 191 F F F 18 360-379 117 117 117 198 198 198 F F F 19 380-399 115 115 115 196 196 196 F F F 20 400-419 114 114 114 195 195 195 F F F 21 420-439 114 114 114 195 195 195 F F F 22 440-459 115 115 115 196 196 196 F F F 23 460-479 113 113 113 194 194 194 F F F 24 480-499 112 112 112 193 193 193 F F F 25 500-519 114 114 114 195 195 195 F F F 26 520-539 116 116 116 197 197 197 F F F 27 540-559 115 115 115 196 196 196 F F F 28 560-579 116 116 116 197 197 197 F F F 29 580-599 115 115 115 196 196 196 F F F Max gen: 117, Max flush: 198 Begin loop 59 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.431499795Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.431865904Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.431899372Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.43193048Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654} {"msg":"Crucible 81c09c40-e4f2-416c-acd9-de91907d054e has session id: 98ae2b0d-5ac6-40e7-8a87-d8135b4631e1","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.431963158Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"Upstairs opts: Upstairs UUID: 81c09c40-e4f2-416c-acd9-de91907d054e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.432013875Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"Crucible stats registered with UUID: 81c09c40-e4f2-416c-acd9-de91907d054e","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.432052413Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.463318353Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654} {"msg":"81c09c40-e4f2-416c-acd9-de91907d054e active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.46335914Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.463393278Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"client":"0","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.463437766Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"client":"0","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.46354744Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"client":"1","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.463594057Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"client":"1","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.463630915Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"client":"2","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.463661573Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"client":"2","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.463783066Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.463820874Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.463848292Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:62264)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.463872701Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:59507)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.4639005Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:44566)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.463925898Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.464316466Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"client":"2","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.464344234Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"client":"2","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.464358703Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"client":"1","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.464372983Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"client":"1","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.464522114Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"client":"0","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.464561032Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"client":"0","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.464980778Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"client":"2","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.465115931Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"client":"1","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.465197676Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"client":"0","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.465520828Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"client":"2","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.465556576Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"client":"2","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.465582024Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.465700518Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"client":"1","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.465733126Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"client":"1","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.465749405Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.465768334Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"client":"0","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.465782313Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"client":"0","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"[0]R flush_numbers[0..12]: [195, 195, 196, 198, 198, 196, 198, 198, 183, 194, 192, 190]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.46582246Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"":"downstairs","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"[0]R generation[0..12]: [114, 114, 115, 117, 117, 115, 117, 117, 107, 113, 112, 111]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.46584055Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"":"downstairs","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.465855089Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"":"downstairs","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"[1]R flush_numbers[0..12]: [195, 195, 196, 198, 198, 196, 198, 198, 183, 194, 192, 190]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.465868898Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"":"downstairs","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"[1]R generation[0..12]: [114, 114, 115, 117, 117, 115, 117, 117, 107, 113, 112, 111]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.465882507Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"":"downstairs","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.465896476Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"":"downstairs","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"[2]R flush_numbers[0..12]: [195, 195, 196, 198, 198, 196, 198, 198, 183, 194, 192, 190]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.465910416Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"":"downstairs","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"[2]R generation[0..12]: [114, 114, 115, 117, 117, 115, 117, 117, 107, 113, 112, 111]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.465924065Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"":"downstairs","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.465937924Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"":"downstairs","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.465950863Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"":"downstairs","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.465966803Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"":"downstairs","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"Next flush: 199","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.465980232Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"":"downstairs","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.465993311Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"":"downstairs","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-26T14:32:25.46600689Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.466024119Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"client":"0","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.466044828Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"client":"1","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.466059047Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"client":"2","":"downstairs","session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.466072477Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.466085006Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"81c09c40-e4f2-416c-acd9-de91907d054e is now active with session: 98ae2b0d-5ac6-40e7-8a87-d8135b4631e1","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.466097955Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.466109964Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654,"session_id":"98ae2b0d-5ac6-40e7-8a87-d8135b4631e1"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:25.466122344Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 97 len 4096 data: 5 02/30 Write block 46 len 28672 data: 10 10 12 11 13 11 12 03/30 Flush 04/30 Flush 05/30 Write block 492 len 8192 data: 10 10 06/30 Write block 517 len 12288 data: 7 9 10 07/30 Read block 372 len 4096 08/30 Write block 212 len 4096 data: 9 09/30 Read block 533 len 24576 10/30 Read block 421 len 4096 11/30 Write block 269 len 4096 data: 12 12/30 Read block 339 len 24576 13/30 Flush 14/30 Write block 32 len 16384 data: 7 7 3 5 15/30 Write block 128 len 16384 data: 8 8 8 8 16/30 Read block 75 len 12288 17/30 Read block 359 len 40960 18/30 Read block 45 len 8192 19/30 Read block 28 len 24576 20/30 Read block 450 len 28672 21/30 Read block 396 len 20480 22/30 Read block 87 len 16384 23/30 Read block 346 len 28672 24/30 Read block 34 len 24576 25/30 Read block 208 len 28672 26/30 Write block 77 len 4096 data: 10 27/30 Read block 395 len 24576 28/30 Write block 526 len 4096 data: 11 29/30 Read block 241 len 20480 30/30 Write block 444 len 8192 data: 11 12 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 14 Acked 1013 Write 4 Done Done Done false 15 Acked 1014 Write 4 Done Done Done false 16 Acked 1015 Read 3 Done Done Done false 17 Acked 1016 Read 10 Done Done Done false 18 Acked 1017 Read 2 Done Done Done false 19 Acked 1018 Read 6 Done Done Done false 20 Acked 1019 Read 7 Done Done Done false 21 Acked 1020 Read 5 Done Done Done false 22 Acked 1021 Read 4 Done Done Done false 23 Acked 1022 Read 7 Done Done Done false 24 Acked 1023 Read 6 Done Done Done false 25 Acked 1024 Read 7 Done Done Done false 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Read 6 Done Done Done false 28 Acked 1027 Write 1 Done Done Done false 29 Acked 1028 Read 5 Done Done Done false 30 Acked 1029 Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:25.487563589Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1654} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 195 195 195 F F F 1 020-039 114 114 114 195 195 195 T T T 2 040-059 118 118 118 199 199 199 F F F 3 060-079 117 117 117 198 198 198 T T T 4 080-099 118 118 118 199 199 199 F F F 5 100-119 115 115 115 196 196 196 F F F 6 120-139 117 117 117 198 198 198 T T T 7 140-159 117 117 117 198 198 198 F F F 8 160-179 107 107 107 183 183 183 F F F 9 180-199 113 113 113 194 194 194 F F F 10 200-219 118 118 118 201 201 201 F F F 11 220-239 111 111 111 190 190 190 F F F 12 240-259 112 112 112 193 193 193 F F F 13 260-279 118 118 118 201 201 201 F F F 14 280-299 117 117 117 198 198 198 F F F 15 300-319 116 116 116 197 197 197 F F F 16 320-339 111 111 111 190 190 190 F F F 17 340-359 112 112 112 191 191 191 F F F 18 360-379 117 117 117 198 198 198 F F F 19 380-399 115 115 115 196 196 196 F F F 20 400-419 114 114 114 195 195 195 F F F 21 420-439 114 114 114 195 195 195 F F F 22 440-459 115 115 115 196 196 196 T T T 23 460-479 113 113 113 194 194 194 F F F 24 480-499 118 118 118 201 201 201 F F F 25 500-519 118 118 118 201 201 201 F F F 26 520-539 116 116 116 197 197 197 T T T 27 540-559 115 115 115 196 196 196 F F F 28 560-579 116 116 116 197 197 197 F F F 29 580-599 115 115 115 196 196 196 F F F Max gen: 118, Max flush: 201 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 195 195 195 F F F 1 020-039 119 119 119 202 202 202 F F F 2 040-059 118 118 118 199 199 199 F F F 3 060-079 119 119 119 202 202 202 F F F 4 080-099 118 118 118 199 199 199 F F F 5 100-119 115 115 115 196 196 196 F F F 6 120-139 119 119 119 202 202 202 F F F 7 140-159 117 117 117 198 198 198 F F F 8 160-179 107 107 107 183 183 183 F F F 9 180-199 113 113 113 194 194 194 F F F 10 200-219 118 118 118 201 201 201 F F F 11 220-239 111 111 111 190 190 190 F F F 12 240-259 112 112 112 193 193 193 F F F 13 260-279 118 118 118 201 201 201 F F F 14 280-299 117 117 117 198 198 198 F F F 15 300-319 116 116 116 197 197 197 F F F 16 320-339 111 111 111 190 190 190 F F F 17 340-359 112 112 112 191 191 191 F F F 18 360-379 117 117 117 198 198 198 F F F 19 380-399 115 115 115 196 196 196 F F F 20 400-419 114 114 114 195 195 195 F F F 21 420-439 114 114 114 195 195 195 F F F 22 440-459 119 119 119 202 202 202 F F F 23 460-479 113 113 113 194 194 194 F F F 24 480-499 118 118 118 201 201 201 F F F 25 500-519 118 118 118 201 201 201 F F F 26 520-539 119 119 119 202 202 202 F F F 27 540-559 115 115 115 196 196 196 F F F 28 560-579 116 116 116 197 197 197 F F F 29 580-599 115 115 115 196 196 196 F F F Max gen: 119, Max flush: 202 Begin loop 60 Downstairs to restart: 2 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.868024325Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.868296459Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.868315188Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.868330667Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662} {"msg":"Crucible a3fc6695-6bec-477e-b477-e58df12253ae has session id: e1f4e251-d023-402b-9cfd-edff8e346929","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.868551375Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"Upstairs opts: Upstairs UUID: a3fc6695-6bec-477e-b477-e58df12253ae, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.868599842Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"Crucible stats registered with UUID: a3fc6695-6bec-477e-b477-e58df12253ae","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.86862893Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.899684932Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662} {"msg":"a3fc6695-6bec-477e-b477-e58df12253ae active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.89971552Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.899744868Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"client":"0","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.899790996Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"client":"0","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.899826874Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"client":"1","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.899861322Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"client":"1","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.89989599Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"client":"2","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.899925938Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"client":"2","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.899953476Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.899986445Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.900017123Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"ds_connection connected from Ok(127.0.0.1:47116)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.900048211Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"ds_connection connected from Ok(127.0.0.1:50017)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.900077949Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"ds_connection connected from Ok(127.0.0.1:55366)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.900269649Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.900686095Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"client":"2","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.900718013Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"client":"2","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.900733022Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"client":"0","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.900746442Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"client":"0","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.900759701Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"client":"1","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.90077271Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"client":"1","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.901404254Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"client":"1","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.901489Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"client":"0","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.901522378Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"client":"2","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.901931264Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"client":"1","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.901960163Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"client":"1","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.901974992Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.901991611Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"client":"0","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.9020054Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"client":"0","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.902018939Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.902102515Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"client":"2","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.902125304Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"client":"2","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"[0]R flush_numbers[0..12]: [195, 202, 199, 202, 199, 196, 202, 198, 183, 194, 201, 190]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.902139923Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"":"downstairs","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"[0]R generation[0..12]: [114, 119, 118, 119, 118, 115, 119, 117, 107, 113, 118, 111]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.902154722Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"":"downstairs","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.902168981Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"":"downstairs","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"[1]R flush_numbers[0..12]: [195, 202, 199, 202, 199, 196, 202, 198, 183, 194, 201, 190]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.90218268Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"":"downstairs","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"[1]R generation[0..12]: [114, 119, 118, 119, 118, 115, 119, 117, 107, 113, 118, 111]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.90219658Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"":"downstairs","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.902210579Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"":"downstairs","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"[2]R flush_numbers[0..12]: [195, 202, 199, 202, 199, 196, 202, 198, 183, 194, 201, 190]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.902224268Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"":"downstairs","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"[2]R generation[0..12]: [114, 119, 118, 119, 118, 115, 119, 117, 107, 113, 118, 111]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.902237877Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"":"downstairs","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.902251716Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"":"downstairs","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.902264546Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"":"downstairs","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.902277585Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"":"downstairs","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"Next flush: 203","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.902290234Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"":"downstairs","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.902302843Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.902321022Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.902333172Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"client":"0","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.902346281Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"client":"1","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.90235926Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"client":"2","":"downstairs","session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.902372619Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.902384739Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"a3fc6695-6bec-477e-b477-e58df12253ae is now active with session: e1f4e251-d023-402b-9cfd-edff8e346929","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.902398388Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.902410217Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662,"session_id":"e1f4e251-d023-402b-9cfd-edff8e346929"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:29.902422347Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 153 len 40960 data: 8 8 10 14 13 11 11 8 7 7 02/30 Write block 479 len 40960 data: 11 10 9 8 8 7 8 8 8 7 03/30 Write block 391 len 36864 data: 7 7 6 5 6 8 12 12 11 04/30 Write block 111 len 36864 data: 13 14 16 13 12 12 11 10 11 05/30 Write block 523 len 16384 data: 11 11 11 12 06/30 Read block 423 len 4096 07/30 Write block 413 len 40960 data: 7 8 8 10 9 9 8 11 11 10 08/30 Write block 220 len 28672 data: 9 8 8 7 9 8 7 09/30 Write block 46 len 28672 data: 11 11 13 12 14 12 13 10/30 Flush 11/30 Write block 399 len 20480 data: 12 11 13 12 9 12/30 Read block 100 len 16384 13/30 Write block 179 len 12288 data: 9 11 9 14/30 Flush 15/30 Flush 16/30 Flush 17/30 Write block 216 len 8192 data: 14 15 18/30 Write block 110 len 4096 data: 9 19/30 Read block 448 len 16384 20/30 Read block 109 len 40960 21/30 Write block 95 len 12288 data: 4 5 6 22/30 Read block 226 len 32768 23/30 Read block 173 len 8192 24/30 Read block 475 len 8192 25/30 Write block 560 len 36864 data: 8 6 7 6 4 7 6 6 9 26/30 Read block 532 len 36864 27/30 Read block 458 len 12288 28/30 Write block 435 len 16384 data: 9 9 8 7 29/30 Write block 353 len 16384 data: 15 13 14 12 30/30 Read block 154 len 4096 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 17 Acked 1016 Write 2 Done Done Done false 18 Acked 1017 Write 1 Done Done Done false 19 Acked 1018 Read 4 Done Done Done false 20 Acked 1019 Read 10 Done Done Done false 21 Acked 1020 Write 3 Done Done Done false 22 Acked 1021 Read 8 Done Done Done false 23 Acked 1022 Read 2 Done Done Done false 24 Acked 1023 Read 2 Done Done Done false 25 Acked 1024 Write 9 Done Done Done false 26 Acked 1025 Read 9 Done Done Done false 27 Acked 1026 Read 3 Done Done Done false 28 Acked 1027 Write 4 Done Done Done false 29 Acked 1028 Write 4 Done Done Done false 30 Acked 1029 Read 1 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 13 14 13 40 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:29.92655177Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1662} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 195 195 195 F F F 1 020-039 119 119 119 202 202 202 F F F 2 040-059 120 120 120 203 203 203 F F F 3 060-079 119 119 119 202 202 202 F F F 4 080-099 118 118 118 199 199 199 T T T 5 100-119 120 120 120 203 203 203 T T T 6 120-139 119 119 119 202 202 202 F F F 7 140-159 120 120 120 203 203 203 F F F 8 160-179 120 120 120 204 204 204 F F F 9 180-199 120 120 120 204 204 204 F F F 10 200-219 118 118 118 201 201 201 T T T 11 220-239 120 120 120 203 203 203 F F F 12 240-259 112 112 112 193 193 193 F F F 13 260-279 118 118 118 201 201 201 F F F 14 280-299 117 117 117 198 198 198 F F F 15 300-319 116 116 116 197 197 197 F F F 16 320-339 111 111 111 190 190 190 F F F 17 340-359 112 112 112 191 191 191 T T T 18 360-379 117 117 117 198 198 198 F F F 19 380-399 120 120 120 204 204 204 F F F 20 400-419 120 120 120 204 204 204 F F F 21 420-439 120 120 120 203 203 203 T T T 22 440-459 119 119 119 202 202 202 F F F 23 460-479 120 120 120 203 203 203 F F F 24 480-499 120 120 120 203 203 203 F F F 25 500-519 118 118 118 201 201 201 F F F 26 520-539 120 120 120 203 203 203 F F F 27 540-559 115 115 115 196 196 196 F F F 28 560-579 116 116 116 197 197 197 T T T 29 580-599 115 115 115 196 196 196 F F F Max gen: 120, Max flush: 204 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 195 195 195 F F F 1 020-039 119 119 119 202 202 202 F F F 2 040-059 120 120 120 203 203 203 F F F 3 060-079 119 119 119 202 202 202 F F F 4 080-099 121 121 121 205 205 205 F F F 5 100-119 121 121 121 205 205 205 F F F 6 120-139 119 119 119 202 202 202 F F F 7 140-159 120 120 120 203 203 203 F F F 8 160-179 120 120 120 204 204 204 F F F 9 180-199 120 120 120 204 204 204 F F F 10 200-219 121 121 121 205 205 205 F F F 11 220-239 120 120 120 203 203 203 F F F 12 240-259 112 112 112 193 193 193 F F F 13 260-279 118 118 118 201 201 201 F F F 14 280-299 117 117 117 198 198 198 F F F 15 300-319 116 116 116 197 197 197 F F F 16 320-339 111 111 111 190 190 190 F F F 17 340-359 121 121 121 205 205 205 F F F 18 360-379 117 117 117 198 198 198 F F F 19 380-399 120 120 120 204 204 204 F F F 20 400-419 120 120 120 204 204 204 F F F 21 420-439 121 121 121 205 205 205 F F F 22 440-459 119 119 119 202 202 202 F F F 23 460-479 120 120 120 203 203 203 F F F 24 480-499 120 120 120 203 203 203 F F F 25 500-519 118 118 118 201 201 201 F F F 26 520-539 120 120 120 203 203 203 F F F 27 540-559 115 115 115 196 196 196 F F F 28 560-579 121 121 121 205 205 205 F F F 29 580-599 115 115 115 196 196 196 F F F Max gen: 121, Max flush: 205 Begin loop 61 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.292238681Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.292589892Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.29262035Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.292651178Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670} {"msg":"Crucible 06ffe1a4-c44c-423b-aaee-fd8030ec2722 has session id: 6a186441-a760-4407-a48a-783838362ea8","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.292672707Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"Upstairs opts: Upstairs UUID: 06ffe1a4-c44c-423b-aaee-fd8030ec2722, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.292709405Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"Crucible stats registered with UUID: 06ffe1a4-c44c-423b-aaee-fd8030ec2722","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.292734933Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.324344633Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670} {"msg":"06ffe1a4-c44c-423b-aaee-fd8030ec2722 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.324373361Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.324388231Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"client":"0","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.324415439Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"client":"0","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.324431168Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"client":"1","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.324444937Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"client":"1","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.324458327Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"client":"2","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.324476006Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"client":"2","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.324501144Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.324528183Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.324553591Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"ds_connection connected from Ok(127.0.0.1:63514)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.324616758Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"ds_connection connected from Ok(127.0.0.1:43837)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.324842465Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"ds_connection connected from Ok(127.0.0.1:39187)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.324877533Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.325515827Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"client":"0","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.325551375Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"client":"0","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.325580443Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"client":"2","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.325608171Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"client":"2","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.32564215Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"client":"1","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.325671128Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"client":"1","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.326090484Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"client":"0","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.326250225Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"client":"1","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.326322361Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"client":"2","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.326716059Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"client":"0","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.326744427Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"client":"0","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.326759646Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.326842672Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"client":"1","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.32687547Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"client":"1","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.326896729Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.326909678Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"client":"2","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.326923637Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"client":"2","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"[0]R flush_numbers[0..12]: [195, 202, 203, 202, 205, 205, 202, 203, 204, 204, 205, 203]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.326937546Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"":"downstairs","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"[0]R generation[0..12]: [114, 119, 120, 119, 121, 121, 119, 120, 120, 120, 121, 120]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.326951426Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"":"downstairs","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.326969765Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"":"downstairs","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"[1]R flush_numbers[0..12]: [195, 202, 203, 202, 205, 205, 202, 203, 204, 204, 205, 203]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.326984344Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"":"downstairs","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"[1]R generation[0..12]: [114, 119, 120, 119, 121, 121, 119, 120, 120, 120, 121, 120]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.326998103Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"":"downstairs","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.327012442Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"":"downstairs","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"[2]R flush_numbers[0..12]: [195, 202, 203, 202, 205, 205, 202, 203, 204, 204, 205, 203]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.327026261Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"":"downstairs","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"[2]R generation[0..12]: [114, 119, 120, 119, 121, 121, 119, 120, 120, 120, 121, 120]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.327040261Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"":"downstairs","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.32705422Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"":"downstairs","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.327067129Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"":"downstairs","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {Wait for a query_work_queue command to finish before sending IO "msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.327087678Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"":"downstairs","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"Next flush: 206","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.327112646Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"":"downstairs","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.327133595Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"":"downstairs","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.327152084Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.327173453Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"client":"0","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.327197521Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"client":"1","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.32722468Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"client":"2","":"downstairs","session_id":"6a186441-a760-4407-a48a-783838362ea8"} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.327253518Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.327276417Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"06ffe1a4-c44c-423b-aaee-fd8030ec2722 is now active with session: 6a186441-a760-4407-a48a-783838362ea8","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.327297366Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.327310115Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670,"session_id":"6a186441-a760-4407-a48a-783838362ea8"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:34.327322445Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 499 len 4096 data: 11 02/30 Write block 94 len 24576 data: 7 5 6 7 6 7 03/30 Read block 276 len 16384 04/30 Write block 248 len 28672 data: 9 10 7 6 7 6 9 05/30 Read block 135 len 28672 06/30 Write block 107 len 16384 data: 10 12 10 10 07/30 Write block 588 len 16384 data: 9 9 8 6 08/30 Write block 430 len 40960 data: 10 8 9 10 8 10 10 9 8 6 09/30 Read block 261 len 4096 10/30 Write block 409 len 24576 data: 2 3 3 5 8 9 11/30 Write block 386 len 40960 data: 7 7 6 6 7 8 8 7 6 7 12/30 Write block 506 len 8192 data: 11 11 13/30 Write block 293 len 20480 data: 10 9 10 9 7 14/30 Flush 15/30 Write block 120 len 20480 data: 11 15 14 12 13 16/30 Write block 383 len 40960 data: 9 8 8 8 8 7 7 8 9 9 17/30 Read block 320 len 8192 18/30 Read block 429 len 20480 19/30 Write block 222 len 36864 data: 9 8 10 9 8 7 8 6 7 20/30 Read block 528 len 16384 21/30 Write block 561 len 4096 data: 7 22/30 Flush 23/30 Write block 111 len 40960 data: 14 15 17 14 13 13 12 11 12 12 24/30 Read block 422 len 12288 25/30 Write block 420 len 28672 data: 12 12 11 10 9 10 9 26/30 Write block 282 len 4096 data: 13 27/30 Write block 408 len 4096 data: 3 28/30 Write block 131 len 32768 data: 9 9 9 8 6 7 7 5 29/30 Write block 42 len 12288 data: 6 8 7 30/30 Read block 220 len 24576 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 10 Done Done Done false 24 Acked 1023 Read 3 Done Done Done false 25 Acked 1024 Write 7 Done Done Done false 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Write 1 Done Done Done false 28 Acked 1027 Write 8 Done Done Done false 29 Acked 1028 Write 3 Done Sent Done false 30 Acked 1029 Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 1 3 Done 8 6 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:34.349616942Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1670} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 195 195 195 F F F 1 020-039 119 119 119 202 202 202 F F F 2 040-059 120 120 120 203 203 203 T T T 3 060-079 119 119 119 202 202 202 F F F 4 080-099 122 122 122 206 206 206 F F F 5 100-119 122 122 122 206 206 206 T T T 6 120-139 122 122 122 207 207 207 T T T 7 140-159 120 120 120 203 203 203 F F F 8 160-179 120 120 120 204 204 204 F F F 9 180-199 120 120 120 204 204 204 F F F 10 200-219 121 121 121 205 205 205 F F F 11 220-239 122 122 122 207 207 207 F F F 12 240-259 122 122 122 206 206 206 F F F 13 260-279 118 118 118 201 201 201 F F F 14 280-299 122 122 122 206 206 206 T T T 15 300-319 116 116 116 197 197 197 F F F 16 320-339 111 111 111 190 190 190 F F F 17 340-359 121 121 121 205 205 205 F F F 18 360-379 117 117 117 198 198 198 F F F 19 380-399 122 122 122 207 207 207 F F F 20 400-419 122 122 122 206 206 206 T T T 21 420-439 122 122 122 206 206 206 T T T 22 440-459 119 119 119 202 202 202 F F F 23 460-479 120 120 120 203 203 203 F F F 24 480-499 122 122 122 206 206 206 F F F 25 500-519 122 122 122 206 206 206 F F F 26 520-539 120 120 120 203 203 203 F F F 27 540-559 115 115 115 196 196 196 F F F 28 560-579 122 122 122 207 207 207 F F F 29 580-599 122 122 122 206 206 206 F F F Max gen: 122, Max flush: 207 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 195 195 195 F F F 1 020-039 119 119 119 202 202 202 F F F 2 040-059 123 123 123 208 208 208 F F F 3 060-079 119 119 119 202 202 202 F F F 4 080-099 122 122 122 206 206 206 F F F 5 100-119 123 123 123 208 208 208 F F F 6 120-139 123 123 123 208 208 208 F F F 7 140-159 120 120 120 203 203 203 F F F 8 160-179 120 120 120 204 204 204 F F F 9 180-199 120 120 120 204 204 204 F F F 10 200-219 121 121 121 205 205 205 F F F 11 220-239 122 122 122 207 207 207 F F F 12 240-259 122 122 122 206 206 206 F F F 13 260-279 118 118 118 201 201 201 F F F 14 280-299 123 123 123 208 208 208 F F F 15 300-319 116 116 116 197 197 197 F F F 16 320-339 111 111 111 190 190 190 F F F 17 340-359 121 121 121 205 205 205 F F F 18 360-379 117 117 117 198 198 198 F F F 19 380-399 122 122 122 207 207 207 F F F 20 400-419 123 123 123 208 208 208 F F F 21 420-439 123 123 123 208 208 208 F F F 22 440-459 119 119 119 202 202 202 F F F 23 460-479 120 120 120 203 203 203 F F F 24 480-499 122 122 122 206 206 206 F F F 25 500-519 122 122 122 206 206 206 F F F 26 520-539 120 120 120 203 203 203 F F F 27 540-559 115 115 115 196 196 196 F F F 28 560-579 122 122 122 207 207 207 F F F 29 580-599 122 122 122 206 206 206 F F F Max gen: 123, Max flush: 208 Begin loop 62 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.71857515Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.71892801Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.718951909Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.718983057Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678} {"msg":"Crucible aee45f9b-1399-47c3-b605-09c4a8fad406 has session id: 87af5040-d48f-476d-9281-62b935f19f1e","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.719025974Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"Upstairs opts: Upstairs UUID: aee45f9b-1399-47c3-b605-09c4a8fad406, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.719065862Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"Crucible stats registered with UUID: aee45f9b-1399-47c3-b605-09c4a8fad406","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.719087411Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.750153482Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678} {"msg":"aee45f9b-1399-47c3-b605-09c4a8fad406 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.75018557Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.750203839Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"client":"0","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.750233857Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"client":"0","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.750265085Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"client":"1","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.750295543Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"client":"1","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.750314402Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"client":"2","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.75034632Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"client":"2","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.750379179Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.750446595Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.750490972Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"ds_connection connected from Ok(127.0.0.1:50508)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.750617335Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"ds_connection connected from Ok(127.0.0.1:41938)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.750655623Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"ds_connection connected from Ok(127.0.0.1:57440)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.750675742Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.75123581Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"client":"2","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.751270788Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"client":"2","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.751383682Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"client":"0","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.751407171Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"client":"0","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.751544073Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"client":"1","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.751565522Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"client":"1","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.751895873Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"client":"2","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.752103961Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"client":"0","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.752257902Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"client":"1","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.752520417Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"client":"2","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.752543456Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"client":"2","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.752557865Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.752644121Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"client":"0","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.752663709Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"client":"0","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.752677188Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.752804361Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"client":"1","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.75282234Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"client":"1","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"[0]R flush_numbers[0..12]: [195, 202, 208, 202, 206, 208, 208, 203, 204, 204, 205, 207]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.752840299Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"":"downstairs","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"[0]R generation[0..12]: [114, 119, 123, 119, 122, 123, 123, 120, 120, 120, 121, 122]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.752854158Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"":"downstairs","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.752868338Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"":"downstairs","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"[1]R flush_numbers[0..12]: [195, 202, 208, 202, 206, 208, 208, 203, 204, 204, 205, 207]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.752882137Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"":"downstairs","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"[1]R generation[0..12]: [114, 119, 123, 119, 122, 123, 123, 120, 120, 120, 121, 122]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.752895706Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"":"downstairs","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.752909526Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"":"downstairs","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"[2]R flush_numbers[0..12]: [195, 202, 208, 202, 206, 208, 208, 203, 204, 204, 205, 207]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.752926465Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"":"downstairs","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"[2]R generation[0..12]: [114, 119, 123, 119, 122, 123, 123, 120, 120, 120, 121, 122]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.752940814Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"":"downstairs","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.752954783Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"":"downstairs","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.752967652Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"":"downstairs","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.752980601Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"":"downstairs","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"Next flush: 209","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.752993341Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"":"downstairs","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.75300686Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"":"downstairs","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-09-26T14:32:38.753020109Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.753035188Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"client":"0","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.753048678Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"client":"1","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.753061927Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"client":"2","":"downstairs","session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.753075346Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.753087525Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"aee45f9b-1399-47c3-b605-09c4a8fad406 is now active with session: 87af5040-d48f-476d-9281-62b935f19f1e","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.753100285Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.753112194Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678,"session_id":"87af5040-d48f-476d-9281-62b935f19f1e"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:38.753126263Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 559 len 40960 data: 7 9 8 8 7 5 8 7 7 10 02/30 Read block 453 len 20480 03/30 Write block 538 len 28672 data: 11 11 11 11 10 11 11 04/30 Write block 567 len 36864 data: 8 11 10 10 14 15 15 15 13 05/30 Read block 356 len 20480 06/30 Read block 292 len 36864 07/30 Read block 119 len 8192 08/30 Flush 09/30 Write block 586 len 32768 data: 11 11 10 10 9 7 6 6 10/30 Write block 285 len 12288 data: 14 13 12 11/30 Read block 409 len 12288 12/30 Write block 514 len 12288 data: 7 5 5 13/30 Write block 480 len 12288 data: 11 10 9 14/30 Read block 251 len 36864 15/30 Read block 195 len 12288 16/30 Flush 17/30 Write block 362 len 24576 data: 13 12 11 8 6 6 18/30 Write block 396 len 32768 data: 9 13 13 13 12 14 13 10 19/30 Read block 242 len 20480 20/30 Read block 260 len 24576 21/30 Read block 290 len 28672 22/30 Write block 206 len 28672 data: 9 7 4 9 10 9 10 23/30 Read block 400 len 32768 24/30 Read block 582 len 32768 25/30 Write block 463 len 8192 data: 6 6 26/30 Read block 543 len 32768 27/30 Write block 435 len 40960 data: 11 11 10 9 7 8 10 9 10 12 28/30 Write block 32 len 20480 data: 8 8 4 6 5 29/30 Read block 102 len 12288 30/30 Write block 403 len 12288 data: 11 7 6 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 17 Acked 1016 Write 6 Done Done Done false 18 Acked 1017 Write 8 Done Done Done false 19 Acked 1018 Read 5 Done Done Done false 20 Acked 1019 Read 6 Done Done Done false 21 Acked 1020 Read 7 Done Done Done false 22 Acked 1021 Write 7 Done Done Done false 23 Acked 1022 Read 8 Done Done Done false 24 Acked 1023 Read 8 Done Done Done false 25 Acked 1024 Write 2 Done Done Done false 26 Acked 1025 Read 8 Done Done Done false 27 Acked 1026 Write 10 Done Done Done false 28 Acked 1027 Write 5 Done Done Done false 29 Acked 1028 Read 3 Done Done Done false 30 Acked 1029 Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:38.777553679Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1678} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 195 195 195 F F F 1 020-039 119 119 119 202 202 202 T T T 2 040-059 123 123 123 208 208 208 F F F 3 060-079 119 119 119 202 202 202 F F F 4 080-099 122 122 122 206 206 206 F F F 5 100-119 123 123 123 208 208 208 F F F 6 120-139 123 123 123 208 208 208 F F F 7 140-159 120 120 120 203 203 203 F F F 8 160-179 120 120 120 204 204 204 F F F 9 180-199 120 120 120 204 204 204 F F F 10 200-219 121 121 121 205 205 205 T T T 11 220-239 122 122 122 207 207 207 F F F 12 240-259 122 122 122 206 206 206 F F F 13 260-279 118 118 118 201 201 201 F F F 14 280-299 124 124 124 210 210 210 F F F 15 300-319 116 116 116 197 197 197 F F F 16 320-339 111 111 111 190 190 190 F F F 17 340-359 121 121 121 205 205 205 F F F 18 360-379 117 117 117 198 198 198 T T T 19 380-399 122 122 122 207 207 207 T T T 20 400-419 123 123 123 208 208 208 T T T 21 420-439 123 123 123 208 208 208 T T T 22 440-459 119 119 119 202 202 202 T T T 23 460-479 120 120 120 203 203 203 T T T 24 480-499 124 124 124 210 210 210 F F F 25 500-519 124 124 124 210 210 210 F F F 26 520-539 124 124 124 209 209 209 F F F 27 540-559 124 124 124 209 209 209 F F F 28 560-579 124 124 124 209 209 209 F F F 29 580-599 124 124 124 210 210 210 F F F Max gen: 124, Max flush: 210 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 195 195 195 F F F 1 020-039 125 125 125 211 211 211 F F F 2 040-059 123 123 123 208 208 208 F F F 3 060-079 119 119 119 202 202 202 F F F 4 080-099 122 122 122 206 206 206 F F F 5 100-119 123 123 123 208 208 208 F F F 6 120-139 123 123 123 208 208 208 F F F 7 140-159 120 120 120 203 203 203 F F F 8 160-179 120 120 120 204 204 204 F F F 9 180-199 120 120 120 204 204 204 F F F 10 200-219 125 125 125 211 211 211 F F F 11 220-239 122 122 122 207 207 207 F F F 12 240-259 122 122 122 206 206 206 F F F 13 260-279 118 118 118 201 201 201 F F F 14 280-299 124 124 124 210 210 210 F F F 15 300-319 116 116 116 197 197 197 F F F 16 320-339 111 111 111 190 190 190 F F F 17 340-359 121 121 121 205 205 205 F F F 18 360-379 125 125 125 211 211 211 F F F 19 380-399 125 125 125 211 211 211 F F F 20 400-419 125 125 125 211 211 211 F F F 21 420-439 125 125 125 211 211 211 F F F 22 440-459 125 125 125 211 211 211 F F F 23 460-479 125 125 125 211 211 211 F F F 24 480-499 124 124 124 210 210 210 F F F 25 500-519 124 124 124 210 210 210 F F F 26 520-539 124 124 124 209 209 209 F F F 27 540-559 124 124 124 209 209 209 F F F 28 560-579 124 124 124 209 209 209 F F F 29 580-599 124 124 124 210 210 210 F F F Max gen: 125, Max flush: 211 Begin loop 63 Downstairs to restart: 2 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.16637862Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.166715531Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.16674809Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.166774918Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686} {"msg":"Crucible 190affc6-1e7e-4610-9a5a-fdaff31e504d has session id: 15e0718a-348d-48d3-8e45-7d82384fc8a5","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.166940319Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"Upstairs opts: Upstairs UUID: 190affc6-1e7e-4610-9a5a-fdaff31e504d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.167009435Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"Crucible stats registered with UUID: 190affc6-1e7e-4610-9a5a-fdaff31e504d","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.167034853Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.198117233Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686} {"msg":"190affc6-1e7e-4610-9a5a-fdaff31e504d active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.198145671Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.19816295Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"client":"0","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.198191669Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"client":"0","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.198209448Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"client":"1","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.198226307Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"client":"1","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.198242576Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"client":"2","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.198259025Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"client":"2","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.198275404Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.198301912Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.19833932Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:34325)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.198371468Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:58061)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.198403337Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:54921)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.19869129Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.199108807Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"client":"1","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.199137245Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"client":"1","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.199275667Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"client":"0","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.199353073Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"client":"0","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.199380261Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"client":"2","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.19939467Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"client":"2","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.199678905Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"client":"1","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.199819746Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"client":"0","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.199925551Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"client":"2","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.200362196Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"client":"1","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.200388654Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"client":"1","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.200407653Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.200421863Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"client":"2","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.200435752Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"client":"2","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.200449291Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.20046148Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"client":"0","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.200475009Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"client":"0","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"[0]R flush_numbers[0..12]: [195, 211, 208, 202, 206, 208, 208, 203, 204, 204, 211, 207]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.200496648Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"":"downstairs","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"[0]R generation[0..12]: [114, 125, 123, 119, 122, 123, 123, 120, 120, 120, 125, 122]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.200511007Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"":"downstairs","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.200525057Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"":"downstairs","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"[1]R flush_numbers[0..12]: [195, 211, 208, 202, 206, 208, 208, 203, 204, 204, 211, 207]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.200539016Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"":"downstairs","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"[1]R generation[0..12]: [114, 125, 123, 119, 122, 123, 123, 120, 120, 120, 125, 122]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.200552755Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"":"downstairs","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.200566894Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"":"downstairs","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"[2]R flush_numbers[0..12]: [195, 211, 208, 202, 206, 208, 208, 203, 204, 204, 211, 207]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.200580614Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"":"downstairs","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"[2]R generation[0..12]: [114, 125, 123, 119, 122, 123, 123, 120, 120, 120, 125, 122]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.200594143Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"":"downstairs","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.200609902Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"":"downstairs","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.200623031Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"":"downstairs","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.20063643Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"":"downstairs","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"Next flush: 212","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.200649359Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"":"downstairs","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.200662149Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"":"downstairs","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.200679768Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.200696477Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"client":"0","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.200710046Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3Wait for a query_work_queue command to finish before sending IO ","pid":1686,"client":"1","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.200727035Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"client":"2","":"downstairs","session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.200740364Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.200752444Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"190affc6-1e7e-4610-9a5a-fdaff31e504d is now active with session: 15e0718a-348d-48d3-8e45-7d82384fc8a5","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.200765303Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.200777222Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686,"session_id":"15e0718a-348d-48d3-8e45-7d82384fc8a5"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:43.200789342Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 28 len 24576 data: 12 7 8 8 9 9 02/30 Read block 123 len 40960 03/30 Read block 35 len 32768 04/30 Read block 63 len 4096 05/30 Read block 80 len 4096 06/30 Read block 253 len 8192 07/30 Write block 476 len 36864 data: 9 7 9 12 12 11 10 9 8 08/30 Write block 487 len 32768 data: 9 8 9 9 9 11 11 12 09/30 Read block 373 len 24576 10/30 Write block 170 len 16384 data: 10 10 9 9 11/30 Write block 584 len 28672 data: 9 11 12 12 11 11 10 12/30 Read block 122 len 16384 13/30 Write block 526 len 32768 data: 13 10 10 8 8 10 11 12 14/30 Read block 523 len 8192 15/30 Read block 461 len 32768 16/30 Write block 447 len 36864 data: 12 10 10 9 8 7 7 8 10 17/30 Write block 84 len 36864 data: 10 10 15 14 12 12 12 12 11 18/30 Write block 547 len 24576 data: 14 14 12 12 10 7 19/30 Read block 32 len 12288 20/30 Write block 93 len 40960 data: 11 8 6 7 8 7 8 9 12 11 21/30 Read block 457 len 24576 22/30 Write block 218 len 36864 data: 12 11 10 9 10 9 11 10 9 23/30 Read block 27 len 40960 24/30 Write block 140 len 24576 data: 7 8 6 7 8 7 25/30 Write block 582 len 28672 data: 8 10 10 12 13 13 12 26/30 Read block 214 len 20480 27/30 Read block 589 len 40960 28/30 Read block 207 len 40960 29/30 Read block 82 len 36864 30/30 Write block 285 len 32768 data: 15 14 13 13 15 14 13 11 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Write 6 Done Done Done false 2 Acked 1001 Read 10 Done Done Done false 3 Acked 1002 Read 8 Done Done Done false 4 Acked 1003 Read 1 Done Done Done false 5 Acked 1004 Read 1 Done Done Done false 6 Acked 1005 Read 2 Done Done Done false 7 Acked 1006 Write 9 Done Done Done false 8 Acked 1007 Write 8 Done Done Done false 9 Acked 1008 Read 6 Done Done Done false 10 Acked 1009 Write 4 Done Done Done false 11 Acked 1010 Write 7 Done Done Done false 12 Acked 1011 Read 4 Done Done Done false 13 Acked 1012 Write 8 Done Done Done false 14 Acked 1013 Read 2 Done Done Done false 15 Acked 1014 Read 8 Done Done Done false 16 Acked 1015 Write 9 Done Done Done false 17 Acked 1016 Write 9 Done Done Done false 18 Acked 1017 Write 6 Done Done Done false 19 Acked 1018 Read 3 Done Done Done false 20 Acked 1019 Write 10 Done Done Done false 21 Acked 1020 Read 6 Done Done Done false 22 Acked 1021 Write 9 Done Done Done false 23 Acked 1022 Read 10 Done Done Done false 24 Acked 1023 Write 6 Done Done Done false 25 Acked 1024 Write 7 Done Done Done false 26 Acked 1025 Read 5 Done Done Done false 27 Acked 1026 Read 10 Done Done Done false 28 Acked 1027 Read 10 Done Done Done false 29 Acked 1028 Read 9 Done Done Done false 30 Acked 1029 Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:43.222772017Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1686} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 195 195 195 F F F 1 020-039 125 125 125 211 211 211 T T T 2 040-059 123 123 123 208 208 208 F F F 3 060-079 119 119 119 202 202 202 F F F 4 080-099 122 122 122 206 206 206 T T T 5 100-119 123 123 123 208 208 208 T T T 6 120-139 123 123 123 208 208 208 F F F 7 140-159 120 120 120 203 203 203 T T T 8 160-179 120 120 120 204 204 204 T T T 9 180-199 120 120 120 204 204 204 F F F 10 200-219 125 125 125 211 211 211 T T T 11 220-239 122 122 122 207 207 207 T T T 12 240-259 122 122 122 206 206 206 F F F 13 260-279 118 118 118 201 201 201 F F F 14 280-299 124 124 124 210 210 210 T T T 15 300-319 116 116 116 197 197 197 F F F 16 320-339 111 111 111 190 190 190 F F F 17 340-359 121 121 121 205 205 205 F F F 18 360-379 125 125 125 211 211 211 F F F 19 380-399 125 125 125 211 211 211 F F F 20 400-419 125 125 125 211 211 211 F F F 21 420-439 125 125 125 211 211 211 F F F 22 440-459 125 125 125 211 211 211 T T T 23 460-479 125 125 125 211 211 211 T T T 24 480-499 124 124 124 210 210 210 T T T 25 500-519 124 124 124 210 210 210 F F F 26 520-539 124 124 124 209 209 209 T T T 27 540-559 124 124 124 209 209 209 T T T 28 560-579 124 124 124 209 209 209 F F F 29 580-599 124 124 124 210 210 210 T T T Max gen: 125, Max flush: 211 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 195 195 195 F F F 1 020-039 126 126 126 212 212 212 F F F 2 040-059 123 123 123 208 208 208 F F F 3 060-079 119 119 119 202 202 202 F F F 4 080-099 126 126 126 212 212 212 F F F 5 100-119 126 126 126 212 212 212 F F F 6 120-139 123 123 123 208 208 208 F F F 7 140-159 126 126 126 212 212 212 F F F 8 160-179 126 126 126 212 212 212 F F F 9 180-199 120 120 120 204 204 204 F F F 10 200-219 126 126 126 212 212 212 F F F 11 220-239 126 126 126 212 212 212 F F F 12 240-259 122 122 122 206 206 206 F F F 13 260-279 118 118 118 201 201 201 F F F 14 280-299 126 126 126 212 212 212 F F F 15 300-319 116 116 116 197 197 197 F F F 16 320-339 111 111 111 190 190 190 F F F 17 340-359 121 121 121 205 205 205 F F F 18 360-379 125 125 125 211 211 211 F F F 19 380-399 125 125 125 211 211 211 F F F 20 400-419 125 125 125 211 211 211 F F F 21 420-439 125 125 125 211 211 211 F F F 22 440-459 126 126 126 212 212 212 F F F 23 460-479 126 126 126 212 212 212 F F F 24 480-499 126 126 126 212 212 212 F F F 25 500-519 124 124 124 210 210 210 F F F 26 520-539 126 126 126 212 212 212 F F F 27 540-559 126 126 126 212 212 212 F F F 28 560-579 124 124 124 209 209 209 F F F 29 580-599 126 126 126 212 212 212 F F F Max gen: 126, Max flush: 212 Begin loop 64 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.721347143Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.72157531Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.721594449Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.721610037Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694} {"msg":"Crucible 25a42d1c-df3f-4fe9-a90d-28106f8ab300 has session id: 4b69f794-22b7-414d-9dba-137f5e50ba4a","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.721777168Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"Upstairs opts: Upstairs UUID: 25a42d1c-df3f-4fe9-a90d-28106f8ab300, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.721830205Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"Crucible stats registered with UUID: 25a42d1c-df3f-4fe9-a90d-28106f8ab300","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.721848444Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.753563628Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694} {"msg":"25a42d1c-df3f-4fe9-a90d-28106f8ab300 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.753599706Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.753622384Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"client":"0","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.753667962Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"client":"0","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.75370403Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"client":"1","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.753738768Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"client":"1","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.753771516Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"client":"2","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.753805594Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"client":"2","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.753842292Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.753895029Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.753927917Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"ds_connection connected from Ok(127.0.0.1:42532)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.753994414Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"ds_connection connected from Ok(127.0.0.1:45870)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.754073519Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"ds_connection connected from Ok(127.0.0.1:60319)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.754104527Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.754663176Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"client":"1","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.754691164Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"client":"1","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.754706453Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"client":"2","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.754720302Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"client":"2","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.754734002Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"client":"0","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.754748561Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"client":"0","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.755400204Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"client":"1","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.755440132Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"client":"2","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.7554713Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"client":"0","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.756097954Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"client":"1","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.756124883Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"client":"1","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.756139602Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.756152101Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"client":"2","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.756165791Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"client":"2","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.75617899Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.756191239Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"client":"0","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.756204358Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"client":"0","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"[0]R flush_numbers[0..12]: [195, 212, 208, 202, 212, 212, 208, 212, 212, 204, 212, 212]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.756218097Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"":"downstairs","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"[0]R generation[0..12]: [114, 126, 123, 119, 126, 126, 123, 126, 126, 120, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.756231897Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"":"downstairs","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.756245976Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"":"downstairs","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"[1]R flush_numbers[0..12]: [195, 212, 208, 202, 212, 212, 208, 212, 212, 204, 212, 212]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.756259605Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"":"downstairs","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"[1]R generation[0..12]: [114, 126, 123, 119, 126, 126, 123, 126, 126, 120, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.756273394Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"":"downstairs","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.756287494Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"":"downstairs","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"[2]R flush_numbers[0..12]: [195, 212, 208, 202, 212, 212, 208, 212, 212, 204, 212, 212]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.756304543Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"":"downstairs","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"[2]R generation[0..12]: [114, 126, 123, 119, 126, 126, 123, 126, 126, 120, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.756318752Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"":"downstairs","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.756332851Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"":"downstairs","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"Max found gen is 127","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.7563513Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"":"downstairs","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"Generation requested: 128 >= found:127","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.756373179Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"":"downstairs","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"Next flush: 213","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.756390318Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"":"downstairs","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.756403827Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"":"downstairs","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.756417446Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.756431565Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"client":"0","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.756448955Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"client":"1","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.756466674Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"client":"2","":"downstairs","session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-09-26T14:32:47.756482953Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.756498962Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"25a42d1c-df3f-4fe9-a90d-28106f8ab300 is now active with session: 4b69f794-22b7-414d-9dba-137f5e50ba4a","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.756511951Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.75652393Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694,"session_id":"4b69f794-22b7-414d-9dba-137f5e50ba4a"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:47.756536179Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 563 len 32768 data: 8 6 9 8 9 12 11 11 02/30 Write block 388 len 8192 data: 8 8 03/30 Read block 43 len 32768 04/30 Write block 398 len 8192 data: 14 14 05/30 Read block 74 len 8192 06/30 Read block 285 len 20480 07/30 Write block 320 len 28672 data: 12 10 7 7 8 10 9 08/30 Write block 167 len 32768 data: 8 7 9 11 11 10 10 8 09/30 Write block 19 len 32768 data: 8 6 5 7 9 10 8 10 10/30 Read block 86 len 36864 11/30 Write block 15 len 20480 data: 9 8 8 8 9 12/30 Write block 188 len 12288 data: 9 7 7 13/30 Read block 359 len 36864 14/30 Write block 193 len 24576 data: 6 7 9 9 8 10 15/30 Read block 80 len 8192 16/30 Read block 120 len 24576 17/30 Write block 434 len 28672 data: 9 12 12 11 10 8 9 18/30 Read block 556 len 20480 19/30 Write block 438 len 12288 data: 11 9 10 20/30 Write block 471 len 12288 data: 8 8 8 21/30 Read block 592 len 4096 22/30 Flush 23/30 Write block 28 len 8192 data: 13 8 24/30 Write block 182 len 36864 data: 12 14 14 12 8 9 10 8 8 25/30 Read block 352 len 40960 26/30 Read block 257 len 36864 27/30 Write block 119 len 12288 data: 13 13 16 28/30 Read block 559 len 32768 29/30 Read block 592 len 28672 30/30 Write block 563 len 28672 data: 9 7 10 9 10 13 12 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 2 Done Done Done false 24 Acked 1023 Write 9 Done Done Done false 25 Acked 1024 Read 10 Done Done Done false 26 Acked 1025 Read 9 Done Done Done false 27 Acked 1026 Write 3 Done Done Done false 28 Acked 1027 Read 8 Done Done Done false 29 Acked 1028 Read 7 Done Done Done false 30 Acked 1029 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:47.779217314Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1694} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 213 213 213 F F F 1 020-039 128 128 128 213 213 213 T T T 2 040-059 123 123 123 208 208 208 F F F 3 060-079 119 119 119 202 202 202 F F F 4 080-099 126 126 126 212 212 212 F F F 5 100-119 126 126 126 212 212 212 T T T 6 120-139 123 123 123 208 208 208 T T T 7 140-159 126 126 126 212 212 212 F F F 8 160-179 128 128 128 213 213 213 F F F 9 180-199 128 128 128 213 213 213 T T T 10 200-219 126 126 126 212 212 212 F F F 11 220-239 126 126 126 212 212 212 F F F 12 240-259 122 122 122 206 206 206 F F F 13 260-279 118 118 118 201 201 201 F F F 14 280-299 126 126 126 212 212 212 F F F 15 300-319 116 116 116 197 197 197 F F F 16 320-339 128 128 128 213 213 213 F F F 17 340-359 121 121 121 205 205 205 F F F 18 360-379 125 125 125 211 211 211 F F F 19 380-399 128 128 128 213 213 213 F F F 20 400-419 125 125 125 211 211 211 F F F 21 420-439 128 128 128 213 213 213 F F F 22 440-459 128 128 128 213 213 213 F F F 23 460-479 128 128 128 213 213 213 F F F 24 480-499 126 126 126 212 212 212 F F F 25 500-519 124 124 124 210 210 210 F F F 26 520-539 126 126 126 212 212 212 F F F 27 540-559 126 126 126 212 212 212 F F F 28 560-579 128 128 128 213 213 213 T T T 29 580-599 126 126 126 212 212 212 F F F Max gen: 128, Max flush: 213 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 213 213 213 F F F 1 020-039 129 129 129 214 214 214 F F F 2 040-059 123 123 123 208 208 208 F F F 3 060-079 119 119 119 202 202 202 F F F 4 080-099 126 126 126 212 212 212 F F F 5 100-119 129 129 129 214 214 214 F F F 6 120-139 129 129 129 214 214 214 F F F 7 140-159 126 126 126 212 212 212 F F F 8 160-179 128 128 128 213 213 213 F F F 9 180-199 129 129 129 214 214 214 F F F 10 200-219 126 126 126 212 212 212 F F F 11 220-239 126 126 126 212 212 212 F F F 12 240-259 122 122 122 206 206 206 F F F 13 260-279 118 118 118 201 201 201 F F F 14 280-299 126 126 126 212 212 212 F F F 15 300-319 116 116 116 197 197 197 F F F 16 320-339 128 128 128 213 213 213 F F F 17 340-359 121 121 121 205 205 205 F F F 18 360-379 125 125 125 211 211 211 F F F 19 380-399 128 128 128 213 213 213 F F F 20 400-419 125 125 125 211 211 211 F F F 21 420-439 128 128 128 213 213 213 F F F 22 440-459 128 128 128 213 213 213 F F F 23 460-479 128 128 128 213 213 213 F F F 24 480-499 126 126 126 212 212 212 F F F 25 500-519 124 124 124 210 210 210 F F F 26 520-539 126 126 126 212 212 212 F F F 27 540-559 126 126 126 212 212 212 F F F 28 560-579 129 129 129 214 214 214 F F F 29 580-599 126 126 126 212 212 212 F F F Max gen: 129, Max flush: 214 Begin loop 65 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.189111092Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.189389606Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.189413045Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.189440663Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702} {"msg":"Crucible 26fdd955-557e-4309-9d01-eaafa3349f51 has session id: 60359e07-e1a3-49e8-b60f-fde718cf62b6","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.189556047Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"Upstairs opts: Upstairs UUID: 26fdd955-557e-4309-9d01-eaafa3349f51, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.189601754Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"Crucible stats registered with UUID: 26fdd955-557e-4309-9d01-eaafa3349f51","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.189623553Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.220648386Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702} {"msg":"26fdd955-557e-4309-9d01-eaafa3349f51 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.220673254Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.220688094Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"client":"0","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.220707883Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"client":"0","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.220721752Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"client":"1","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.220735341Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"client":"1","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.22074866Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"client":"2","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.220764599Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"client":"2","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.220777809Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.220791928Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.220805897Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"ds_connection connected from Ok(127.0.0.1:57831)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.220996596Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"ds_connection connected from Ok(127.0.0.1:65483)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.221206264Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"ds_connection connected from Ok(127.0.0.1:44665)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.221259781Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.221725975Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"client":"1","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.221764043Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"client":"1","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.221928334Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"client":"0","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.221959612Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"client":"0","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.221975701Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"client":"2","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.22198989Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"client":"2","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.222397547Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"client":"1","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.22252444Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"client":"0","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.222639493Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"client":"2","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.223008022Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"client":"1","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.223035961Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"client":"1","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.22305169Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.223137015Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"client":"0","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.223158154Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"client":"0","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.223173343Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.223185902Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"client":"2","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.223199571Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"client":"2","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"[0]R flush_numbers[0..12]: [213, 214, 208, 202, 212, 214, 214, 212, 213, 214, 212, 212]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.223213741Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"":"downstairs","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"[0]R generation[0..12]: [128, 129, 123, 119, 126, 129, 129, 126, 128, 129, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.22322814Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"":"downstairs","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.223242909Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"":"downstairs","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"[1]R flush_numbers[0..12]: [213, 214, 208, 202, 212, 214, 214, 212, 213, 214, 212, 212]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.223257198Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"":"downstairs","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"[1]R generation[0..12]: [128, 129, 123, 119, 126, 129, 129, 126, 128, 129, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.223271027Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"":"downstairs","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.223285187Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"":"downstairs","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"[2]R flush_numbers[0..12]: [213, 214, 208, 202, 212, 214, 214, 212, 213, 214, 212, 212]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.223299026Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"":"downstairs","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"[2]R generation[0..12]: [128, 129, 123, 119, 126, 129, 129, 126, 128, 129, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.223312945Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"":"downstairs","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.223326974Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"":"downstairs","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.223342363Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"":"downstairs","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.223356023Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"":"downstairs","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"Next flush: 215","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.223368982Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"":"downstairs","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.223382781Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"":"downstairs","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":Wait for a query_work_queue command to finish before sending IO "No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.22339959Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.22341195Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"client":"0","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.223425299Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"client":"1","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.223438748Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"client":"2","":"downstairs","session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.223452037Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.223464336Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"26fdd955-557e-4309-9d01-eaafa3349f51 is now active with session: 60359e07-e1a3-49e8-b60f-fde718cf62b6","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.223477286Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.223489355Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702,"session_id":"60359e07-e1a3-49e8-b60f-fde718cf62b6"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:52.223501544Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 546 len 32768 data: 15 15 15 13 13 11 8 7 02/30 Write block 548 len 24576 data: 16 14 14 12 9 8 03/30 Flush 04/30 Flush 05/30 Write block 356 len 20480 data: 13 14 14 12 13 06/30 Write block 124 len 28672 data: 14 14 9 9 9 9 9 07/30 Read block 521 len 36864 08/30 Flush 09/30 Write block 302 len 24576 data: 11 15 15 17 13 13 10/30 Write block 109 len 4096 data: 11 11/30 Write block 350 len 12288 data: 10 12 13 12/30 Write block 406 len 40960 data: 7 3 4 3 4 4 6 9 10 9 13/30 Write block 305 len 4096 data: 18 14/30 Read block 78 len 20480 15/30 Read block 508 len 12288 16/30 Flush 17/30 Write block 171 len 20480 data: 12 11 11 9 8 18/30 Write block 504 len 8192 data: 14 12 19/30 Read block 241 len 40960 20/30 Read block 34 len 16384 21/30 Read block 333 len 20480 22/30 Read block 298 len 16384 23/30 Read block 236 len 28672 24/30 Write block 285 len 4096 data: 16 25/30 Read block 511 len 20480 26/30 Read block 566 len 24576 27/30 Write block 359 len 24576 data: 13 14 12 14 13 12 28/30 Write block 312 len 24576 data: 8 12 14 13 15 15 29/30 Write block 301 len 12288 data: 6 12 16 30/30 Write block 114 len 24576 data: 15 14 14 13 12 14 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 17 Acked 1016 Write 5 Done Done Done false 18 Acked 1017 Write 2 Done Done Done false 19 Acked 1018 Read 10 Done Done Done false 20 Acked 1019 Read 4 Done Done Done false 21 Acked 1020 Read 5 Done Done Done false 22 Acked 1021 Read 4 Done Done Done false 23 Acked 1022 Read 7 Done Done Done false 24 Acked 1023 Write 1 Done Done Done false 25 Acked 1024 Read 5 Done Done Done false 26 Acked 1025 Read 6 Done Done Done false 27 Acked 1026 Write 6 Done Sent Sent false 28 Acked 1027 Write 6 Sent Sent Sent false 29 Acked 1028 Write 3 Sent Sent Sent false 30 Acked 1029 Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 4 4 11 Done 11 10 10 31 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:52.246541629Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1702} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 213 213 213 F F F 1 020-039 129 129 129 214 214 214 F F F 2 040-059 123 123 123 208 208 208 F F F 3 060-079 119 119 119 202 202 202 F F F 4 080-099 126 126 126 212 212 212 F F F 5 100-119 130 130 130 218 218 218 T T T 6 120-139 130 130 130 217 217 217 F F F 7 140-159 126 126 126 212 212 212 F F F 8 160-179 128 128 128 213 213 213 T T T 9 180-199 129 129 129 214 214 214 F F F 10 200-219 126 126 126 212 212 212 F F F 11 220-239 126 126 126 212 212 212 F F F 12 240-259 122 122 122 206 206 206 F F F 13 260-279 118 118 118 201 201 201 F F F 14 280-299 126 126 126 212 212 212 T T T 15 300-319 130 130 130 218 218 218 T T T 16 320-339 128 128 128 213 213 213 F F F 17 340-359 130 130 130 218 218 218 T T T 18 360-379 130 130 130 217 217 217 T T T 19 380-399 128 128 128 213 213 213 F F F 20 400-419 130 130 130 218 218 218 F F F 21 420-439 128 128 128 213 213 213 F F F 22 440-459 128 128 128 213 213 213 F F F 23 460-479 128 128 128 213 213 213 F F F 24 480-499 126 126 126 212 212 212 F F F 25 500-519 124 124 124 210 210 210 T T T 26 520-539 126 126 126 212 212 212 F F F 27 540-559 130 130 130 215 215 215 F F F 28 560-579 129 129 129 214 214 214 F F F 29 580-599 126 126 126 212 212 212 F F F Max gen: 130, Max flush: 218 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 213 213 213 F F F 1 020-039 129 129 129 214 214 214 F F F 2 040-059 123 123 123 208 208 208 F F F 3 060-079 119 119 119 202 202 202 F F F 4 080-099 126 126 126 212 212 212 F F F 5 100-119 131 131 131 219 219 219 F F F 6 120-139 130 130 130 217 217 217 F F F 7 140-159 126 126 126 212 212 212 F F F 8 160-179 131 131 131 219 219 219 F F F 9 180-199 129 129 129 214 214 214 F F F 10 200-219 126 126 126 212 212 212 F F F 11 220-239 126 126 126 212 212 212 F F F 12 240-259 122 122 122 206 206 206 F F F 13 260-279 118 118 118 201 201 201 F F F 14 280-299 131 131 131 219 219 219 F F F 15 300-319 131 131 131 219 219 219 F F F 16 320-339 128 128 128 213 213 213 F F F 17 340-359 131 131 131 219 219 219 F F F 18 360-379 131 131 131 219 219 219 F F F 19 380-399 128 128 128 213 213 213 F F F 20 400-419 130 130 130 218 218 218 F F F 21 420-439 128 128 128 213 213 213 F F F 22 440-459 128 128 128 213 213 213 F F F 23 460-479 128 128 128 213 213 213 F F F 24 480-499 126 126 126 212 212 212 F F F 25 500-519 131 131 131 219 219 219 F F F 26 520-539 126 126 126 212 212 212 F F F 27 540-559 130 130 130 215 215 215 F F F 28 560-579 129 129 129 214 214 214 F F F 29 580-599 126 126 126 212 212 212 F F F Max gen: 131, Max flush: 219 Begin loop 66 Downstairs to restart: 2 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.655147141Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.655353609Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.655369948Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.655390237Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710} {"msg":"Crucible f49fd46d-19d9-4bb1-bc93-591a17f7bea3 has session id: 66865283-fde9-4c27-b131-d75c4eb24654","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.655539099Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"Upstairs opts: Upstairs UUID: f49fd46d-19d9-4bb1-bc93-591a17f7bea3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.655589156Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"Crucible stats registered with UUID: f49fd46d-19d9-4bb1-bc93-591a17f7bea3","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.655611715Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.686432169Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710} {"msg":"f49fd46d-19d9-4bb1-bc93-591a17f7bea3 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.686467117Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.686485076Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"client":"0","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.686526194Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"client":"0","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.686560512Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"client":"1","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.68659567Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"client":"1","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.686635688Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"client":"2","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.686705934Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"client":"2","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.686737762Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.68676913Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.686799028Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"ds_connection connected from Ok(127.0.0.1:47622)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.686831236Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"ds_connection connected from Ok(127.0.0.1:49781)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.68695444Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"ds_connection connected from Ok(127.0.0.1:52676)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.687063174Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.687510388Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"client":"1","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.687541886Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"client":"1","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.687557075Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"client":"2","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.687570685Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"client":"2","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.687583914Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"client":"0","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.687598683Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"client":"0","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.688162771Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"client":"1","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.688296004Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"client":"0","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.688332702Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"client":"2","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.688692421Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"client":"1","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.688728379Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"client":"1","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.688755278Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.688779766Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"client":"0","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.688805375Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"client":"0","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.688832273Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.688854052Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"client":"2","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.688879771Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"client":"2","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"[0]R flush_numbers[0..12]: [213, 214, 208, 202, 212, 219, 217, 212, 219, 214, 212, 212]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.688904449Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"":"downstairs","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"[0]R generation[0..12]: [128, 129, 123, 119, 126, 131, 130, 126, 131, 129, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.688929698Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"":"downstairs","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.688954696Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"":"downstairs","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"[1]R flush_numbers[0..12]: [213, 214, 208, 202, 212, 219, 217, 212, 219, 214, 212, 212]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.688978095Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"":"downstairs","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"[1]R generation[0..12]: [128, 129, 123, 119, 126, 131, 130, 126, 131, 129, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.689000844Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"":"downstairs","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.689026042Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"":"downstairs","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"[2]R flush_numbers[0..12]: [213, 214, 208, 202, 212, 219, 217, 212, 219, 214, 212, 212]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.689053041Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"":"downstairs","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"[2]R generation[0..12]: [128, 129, 123, 119, 126, 131, 130, 126, 131, 129, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.689077589Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"":"downstairs","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.689103418Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"":"downstairs","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.689128846Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.689159115Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"":"downstairs","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"Next flush: 220","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.689185593Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"":"downstairs","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.689210932Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"":"downstairs","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.68924192Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.689268749Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"client":"0","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.689296597Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"client":"1","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-09-26T14:32:56.689323506Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"client":"2","":"downstairs","session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.689350834Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.689370083Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"f49fd46d-19d9-4bb1-bc93-591a17f7bea3 is now active with session: 66865283-fde9-4c27-b131-d75c4eb24654","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.689384322Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.689396611Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710,"session_id":"66865283-fde9-4c27-b131-d75c4eb24654"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:32:56.68940897Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 511 len 24576 data: 8 9 7 8 6 6 03/30 Write block 139 len 36864 data: 6 8 9 7 8 9 8 7 9 04/30 Write block 120 len 24576 data: 14 17 15 13 15 15 05/30 Read block 350 len 24576 06/30 Write block 89 len 8192 data: 13 13 07/30 Write block 442 len 32768 data: 10 11 13 13 12 13 11 11 08/30 Read block 434 len 20480 09/30 Read block 299 len 20480 10/30 Read block 311 len 16384 11/30 Write block 35 len 16384 data: 7 6 5 5 12/30 Read block 285 len 8192 13/30 Flush 14/30 Write block 70 len 24576 data: 8 10 11 11 11 9 15/30 Read block 12 len 24576 16/30 Read block 33 len 28672 17/30 Read block 96 len 36864 18/30 Read block 374 len 36864 19/30 Read block 199 len 12288 20/30 Write block 569 len 20480 data: 13 12 15 16 16 21/30 Read block 62 len 28672 22/30 Read block 291 len 36864 23/30 Read block 135 len 36864 24/30 Read block 118 len 12288 25/30 Read block 298 len 20480 26/30 Read block 296 len 16384 27/30 Read block 68 len 36864 28/30 Read block 335 len 36864 29/30 Read block 296 len 36864 30/30 Write block 38 len 20480 data: 6 6 6 5 7 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 14 Acked 1013 Write 6 Done Done Done false 15 Acked 1014 Read 6 Done Done Done false 16 Acked 1015 Read 7 Done Done Done false 17 Acked 1016 Read 9 Done Done Done false 18 Acked 1017 Read 9 Done Done Done false 19 Acked 1018 Read 3 Done Done Done false 20 Acked 1019 Write 5 Done Done Done false 21 Acked 1020 Read 7 Done Done Done false 22 Acked 1021 Read 9 Done Done Done false 23 Acked 1022 Read 9 Done Done Done false 24 Acked 1023 Read 3 Done Done Done false 25 Acked 1024 Read 5 Done Done Done false 26 Acked 1025 Read 4 Done Done Done false 27 Acked 1026 Read 9 Done Done Done false 28 Acked 1027 Read 9 Done Done Done false 29 Acked 1028 Read 9 Done Done Done false 30 Acked 1029 Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:32:56.712342522Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1710} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 213 213 213 F F F 1 020-039 132 132 132 221 221 221 T T T 2 040-059 123 123 123 208 208 208 T T T 3 060-079 119 119 119 202 202 202 T T T 4 080-099 132 132 132 221 221 221 F F F 5 100-119 131 131 131 219 219 219 F F F 6 120-139 132 132 132 221 221 221 F F F 7 140-159 132 132 132 221 221 221 F F F 8 160-179 131 131 131 219 219 219 F F F 9 180-199 129 129 129 214 214 214 F F F 10 200-219 126 126 126 212 212 212 F F F 11 220-239 126 126 126 212 212 212 F F F 12 240-259 122 122 122 206 206 206 F F F 13 260-279 118 118 118 201 201 201 F F F 14 280-299 131 131 131 219 219 219 F F F 15 300-319 131 131 131 219 219 219 F F F 16 320-339 128 128 128 213 213 213 F F F 17 340-359 131 131 131 219 219 219 F F F 18 360-379 131 131 131 219 219 219 F F F 19 380-399 128 128 128 213 213 213 F F F 20 400-419 130 130 130 218 218 218 F F F 21 420-439 128 128 128 213 213 213 F F F 22 440-459 132 132 132 221 221 221 F F F 23 460-479 128 128 128 213 213 213 F F F 24 480-499 126 126 126 212 212 212 F F F 25 500-519 132 132 132 221 221 221 F F F 26 520-539 126 126 126 212 212 212 F F F 27 540-559 130 130 130 215 215 215 F F F 28 560-579 129 129 129 214 214 214 T T T 29 580-599 126 126 126 212 212 212 F F F Max gen: 132, Max flush: 221 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 213 213 213 F F F 1 020-039 133 133 133 222 222 222 F F F 2 040-059 133 133 133 222 222 222 F F F 3 060-079 133 133 133 222 222 222 F F F 4 080-099 132 132 132 221 221 221 F F F 5 100-119 131 131 131 219 219 219 F F F 6 120-139 132 132 132 221 221 221 F F F 7 140-159 132 132 132 221 221 221 F F F 8 160-179 131 131 131 219 219 219 F F F 9 180-199 129 129 129 214 214 214 F F F 10 200-219 126 126 126 212 212 212 F F F 11 220-239 126 126 126 212 212 212 F F F 12 240-259 122 122 122 206 206 206 F F F 13 260-279 118 118 118 201 201 201 F F F 14 280-299 131 131 131 219 219 219 F F F 15 300-319 131 131 131 219 219 219 F F F 16 320-339 128 128 128 213 213 213 F F F 17 340-359 131 131 131 219 219 219 F F F 18 360-379 131 131 131 219 219 219 F F F 19 380-399 128 128 128 213 213 213 F F F 20 400-419 130 130 130 218 218 218 F F F 21 420-439 128 128 128 213 213 213 F F F 22 440-459 132 132 132 221 221 221 F F F 23 460-479 128 128 128 213 213 213 F F F 24 480-499 126 126 126 212 212 212 F F F 25 500-519 132 132 132 221 221 221 F F F 26 520-539 126 126 126 212 212 212 F F F 27 540-559 130 130 130 215 215 215 F F F 28 560-579 133 133 133 222 222 222 F F F 29 580-599 126 126 126 212 212 212 F F F Max gen: 133, Max flush: 222 Begin loop 67 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.089469125Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.0897341Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.089808676Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.089845214Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718} {"msg":"Crucible 62b0ed4d-2a38-4add-9246-f721ea4d4b6e has session id: d6ceb4ed-d5ae-4992-9023-27401568f53e","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.090002415Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"Upstairs opts: Upstairs UUID: 62b0ed4d-2a38-4add-9246-f721ea4d4b6e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.090046932Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"Crucible stats registered with UUID: 62b0ed4d-2a38-4add-9246-f721ea4d4b6e","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.090070241Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.121702Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718} {"msg":"62b0ed4d-2a38-4add-9246-f721ea4d4b6e active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.121741757Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.121766156Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"client":"0","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.121810023Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"client":"0","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.121843652Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"client":"1","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.12187795Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"client":"1","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.121910998Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"client":"2","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.121941986Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"client":"2","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.121972604Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.121998883Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.122024601Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"ds_connection connected from Ok(127.0.0.1:44016)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.122053709Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"ds_connection connected from Ok(127.0.0.1:39519)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.122081228Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"ds_connection connected from Ok(127.0.0.1:65216)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.12222137Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.122721982Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"client":"2","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.122765179Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"client":"2","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.122795798Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"client":"1","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.122829886Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"client":"1","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.122919231Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"client":"0","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.122946059Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"client":"0","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.123423992Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"client":"2","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.12346022Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"client":"1","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.123496138Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"client":"0","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.123970511Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"client":"2","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.12399623Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"client":"2","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.124010799Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.124076115Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"client":"1","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.124103453Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"client":"1","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.124117973Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.124130582Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"client":"0","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.124144211Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"client":"0","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"[0]R flush_numbers[0..12]: [213, 222, 222, 222, 221, 219, 221, 221, 219, 214, 212, 212]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.12415828Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"":"downstairs","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"[0]R generation[0..12]: [128, 133, 133, 133, 132, 131, 132, 132, 131, 129, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.124175799Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"":"downstairs","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.124190659Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"":"downstairs","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"[1]R flush_numbers[0..12]: [213, 222, 222, 222, 221, 219, 221, 221, 219, 214, 212, 212]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.124204818Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"":"downstairs","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"[1]R generation[0..12]: [128, 133, 133, 133, 132, 131, 132, 132, 131, 129, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.124218777Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"":"downstairs","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.124232956Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"":"downstairs","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"[2]R flush_numbers[0..12]: [213, 222, 222, 222, 221, 219, 221, 221, 219, 214, 212, 212]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.124246925Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"":"downstairs","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"[2]R generation[0..12]: [128, 133, 133, 133, 132, 131, 132, 132, 131, 129, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.124260764Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"":"downstairs","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.124274654Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"":"downstairs","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.124287613Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"":"downstairs","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.124300812Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"":"downstairs","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"Next flush: 223","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.124313792Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"":"downstairs","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-26T14:33:01.124326671Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"":"downstairs","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.12434139Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.124353679Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"client":"0","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.124368998Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"client":"1","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.124382748Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"client":"2","":"downstairs","session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.124396277Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.124408616Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"62b0ed4d-2a38-4add-9246-f721ea4d4b6e is now active with session: d6ceb4ed-d5ae-4992-9023-27401568f53e","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.124421475Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.124433455Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718,"session_id":"d6ceb4ed-d5ae-4992-9023-27401568f53e"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:01.124445654Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 90 len 24576 data: 14 13 12 12 9 7 02/30 Flush 03/30 Write block 172 len 4096 data: 12 04/30 Write block 143 len 20480 data: 9 10 9 8 10 05/30 Read block 498 len 24576 06/30 Write block 374 len 28672 data: 8 8 6 7 9 7 8 07/30 Write block 223 len 28672 data: 10 12 11 10 8 9 7 08/30 Flush 09/30 Flush 10/30 Write block 298 len 4096 data: 12 11/30 Write block 94 len 8192 data: 10 8 12/30 Write block 26 len 20480 data: 11 11 14 9 9 13/30 Write block 546 len 36864 data: 16 16 17 15 15 13 10 9 5 14/30 Read block 585 len 24576 15/30 Read block 186 len 4096 16/30 Write block 136 len 24576 data: 8 8 6 7 9 10 17/30 Read block 525 len 8192 18/30 Write block 192 len 8192 data: 9 7 19/30 Read block 177 len 16384 20/30 Flush 21/30 Write block 387 len 8192 data: 9 9 22/30 Read block 109 len 20480 23/30 Read block 597 len 8192 24/30 Write block 233 len 4096 data: 4 25/30 Write block 445 len 8192 data: 14 13 26/30 Read block 589 len 12288 27/30 Read block 68 len 20480 28/30 Read block 106 len 32768 29/30 Read block 41 len 16384 30/30 Write block 9 len 8192 data: 7 9 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 2 Done Done Done false 22 Acked 1021 Read 5 Done Done Done false 23 Acked 1022 Read 2 Done Done Done false 24 Acked 1023 Write 1 Done Done Done false 25 Acked 1024 Write 2 Done Done Done false 26 Acked 1025 Read 3 Done Done Done false 27 Acked 1026 Read 5 Done Done Done false 28 Acked 1027 Read 8 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:33:01.147858088Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1718} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 213 213 213 T T T 1 020-039 134 134 134 226 226 226 F F F 2 040-059 133 133 133 222 222 222 F F F 3 060-079 133 133 133 222 222 222 F F F 4 080-099 134 134 134 226 226 226 F F F 5 100-119 131 131 131 219 219 219 F F F 6 120-139 134 134 134 226 226 226 F F F 7 140-159 134 134 134 226 226 226 F F F 8 160-179 134 134 134 224 224 224 F F F 9 180-199 134 134 134 226 226 226 F F F 10 200-219 126 126 126 212 212 212 F F F 11 220-239 134 134 134 224 224 224 T T T 12 240-259 122 122 122 206 206 206 F F F 13 260-279 118 118 118 201 201 201 F F F 14 280-299 134 134 134 226 226 226 F F F 15 300-319 131 131 131 219 219 219 F F F 16 320-339 128 128 128 213 213 213 F F F 17 340-359 131 131 131 219 219 219 F F F 18 360-379 134 134 134 224 224 224 F F F 19 380-399 134 134 134 224 224 224 T T T 20 400-419 130 130 130 218 218 218 F F F 21 420-439 128 128 128 213 213 213 F F F 22 440-459 132 132 132 221 221 221 T T T 23 460-479 128 128 128 213 213 213 F F F 24 480-499 126 126 126 212 212 212 F F F 25 500-519 132 132 132 221 221 221 F F F 26 520-539 126 126 126 212 212 212 F F F 27 540-559 134 134 134 226 226 226 F F F 28 560-579 133 133 133 222 222 222 F F F 29 580-599 126 126 126 212 212 212 F F F Max gen: 134, Max flush: 226 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 135 135 135 227 227 227 F F F 1 020-039 134 134 134 226 226 226 F F F 2 040-059 133 133 133 222 222 222 F F F 3 060-079 133 133 133 222 222 222 F F F 4 080-099 134 134 134 226 226 226 F F F 5 100-119 131 131 131 219 219 219 F F F 6 120-139 134 134 134 226 226 226 F F F 7 140-159 134 134 134 226 226 226 F F F 8 160-179 134 134 134 224 224 224 F F F 9 180-199 134 134 134 226 226 226 F F F 10 200-219 126 126 126 212 212 212 F F F 11 220-239 135 135 135 227 227 227 F F F 12 240-259 122 122 122 206 206 206 F F F 13 260-279 118 118 118 201 201 201 F F F 14 280-299 134 134 134 226 226 226 F F F 15 300-319 131 131 131 219 219 219 F F F 16 320-339 128 128 128 213 213 213 F F F 17 340-359 131 131 131 219 219 219 F F F 18 360-379 134 134 134 224 224 224 F F F 19 380-399 135 135 135 227 227 227 F F F 20 400-419 130 130 130 218 218 218 F F F 21 420-439 128 128 128 213 213 213 F F F 22 440-459 135 135 135 227 227 227 F F F 23 460-479 128 128 128 213 213 213 F F F 24 480-499 126 126 126 212 212 212 F F F 25 500-519 132 132 132 221 221 221 F F F 26 520-539 126 126 126 212 212 212 F F F 27 540-559 134 134 134 226 226 226 F F F 28 560-579 133 133 133 222 222 222 F F F 29 580-599 126 126 126 212 212 212 F F F Max gen: 135, Max flush: 227 Begin loop 68 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.512945665Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.513259017Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.513282126Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.513307924Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726} {"msg":"Crucible f0eb845c-4783-401d-91ae-42d6b5147757 has session id: f3a36865-cdb5-4f77-b52f-5866c391813f","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.513499823Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"Upstairs opts: Upstairs UUID: f0eb845c-4783-401d-91ae-42d6b5147757, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.51356398Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"Crucible stats registered with UUID: f0eb845c-4783-401d-91ae-42d6b5147757","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.513599348Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.544557684Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726} {"msg":"f0eb845c-4783-401d-91ae-42d6b5147757 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.544587873Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.544605132Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"client":"0","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.54463253Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"client":"0","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.544662609Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"client":"1","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.544694667Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"client":"1","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.544733254Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"client":"2","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.544767533Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"client":"2","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.544797361Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.544825879Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.545042757Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"ds_connection connected from Ok(127.0.0.1:33054)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.545083355Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"ds_connection connected from Ok(127.0.0.1:61020)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.545109943Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"ds_connection connected from Ok(127.0.0.1:43938)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.545136652Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.545480742Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"client":"0","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.54551001Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"client":"0","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.54552609Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"client":"2","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.545539889Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"client":"2","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.545767576Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"client":"1","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.545804154Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"client":"1","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.54623406Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"client":"0","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.546265848Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"client":"2","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.546350473Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"client":"1","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.546834186Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"client":"2","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.546862364Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"client":"2","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.546878453Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.546958528Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"client":"1","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.546978398Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"client":"1","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.546992467Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.547070142Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"client":"0","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.547088231Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"client":"0","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"[0]R flush_numbers[0..12]: [227, 226, 222, 222, 226, 219, 226, 226, 224, 226, 212, 227]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.54710434Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"":"downstairs","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"[0]R generation[0..12]: [135, 134, 133, 133, 134, 131, 134, 134, 134, 134, 126, 135]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.54711881Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"":"downstairs","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.547133209Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"":"downstairs","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"[1]R flush_numbers[0..12]: [227, 226, 222, 222, 226, 219, 226, 226, 224, 226, 212, 227]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.547147228Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"":"downstairs","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"[1]R generation[0..12]: [135, 134, 133, 133, 134, 131, 134, 134, 134, 134, 126, 135]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.547160907Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"":"downstairs","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.547178616Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"":"downstairs","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"[2]R flush_numbers[0..12]: [227, 226, 222, 222, 226, 219, 226, 226, 224, 226, 212, 227]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.547192615Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"":"downstairs","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"[2]R generation[0..12]: [135, 134, 133, 133, 134, 131, 134, 134, 134, 134, 126, 135]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.547206254Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"":"downstairs","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.547219994Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"":"downstairs","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.547232993Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"":"downstairs","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.547246092Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"":"downstairs","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"Next flush: 228","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.547259102Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"":"downstairs","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.547272051Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"":"downstairs","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"No downstairs reconciliation required","v"Wait for a query_work_queue command to finish before sending IO :0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.54728909Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.547301259Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"client":"0","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.547314668Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"client":"1","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.547327748Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"client":"2","":"downstairs","session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.547341007Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.547353356Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"f0eb845c-4783-401d-91ae-42d6b5147757 is now active with session: f3a36865-cdb5-4f77-b52f-5866c391813f","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.547366316Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.547380905Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726,"session_id":"f3a36865-cdb5-4f77-b52f-5866c391813f"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:05.547393334Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 523 len 40960 data: 12 12 12 14 11 11 9 9 11 12 02/30 Write block 311 len 36864 data: 8 9 13 15 14 16 16 10 12 03/30 Read block 298 len 8192 04/30 Read block 80 len 24576 05/30 Write block 224 len 24576 data: 13 12 11 9 10 8 06/30 Write block 270 len 20480 data: 9 8 8 7 10 07/30 Read block 118 len 8192 08/30 Read block 399 len 16384 09/30 Read block 453 len 8192 10/30 Read block 263 len 12288 11/30 Read block 299 len 20480 12/30 Read block 61 len 36864 13/30 Write block 274 len 12288 data: 11 13 14 14/30 Read block 429 len 12288 15/30 Read block 379 len 40960 16/30 Read block 355 len 12288 17/30 Read block 567 len 28672 18/30 Read block 73 len 20480 19/30 Write block 146 len 28672 data: 9 11 6 6 5 8 8 20/30 Write block 490 len 40960 data: 10 10 12 12 13 13 12 12 12 12 21/30 Write block 64 len 36864 data: 9 9 12 11 11 8 9 11 12 22/30 Write block 167 len 12288 data: 9 8 10 23/30 Write block 430 len 28672 data: 11 9 10 11 10 13 13 24/30 Write block 199 len 36864 data: 6 8 7 8 7 10 10 10 8 25/30 Read block 573 len 4096 26/30 Write block 533 len 32768 data: 13 11 10 11 12 12 12 12 27/30 Read block 567 len 24576 28/30 Write block 374 len 20480 data: 9 9 7 8 10 29/30 Write block 496 len 40960 data: 13 13 13 13 11 13 14 14 15 13 30/30 Write block 334 len 20480 data: 7 5 6 4 5 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Write 10 Done Done Done false 2 Acked 1001 Write 9 Done Done Done false 3 Acked 1002 Read 2 Done Done Done false 4 Acked 1003 Read 6 Done Done Done false 5 Acked 1004 Write 6 Done Done Done false 6 Acked 1005 Write 5 Done Done Done false 7 Acked 1006 Read 2 Done Done Done false 8 Acked 1007 Read 4 Done Done Done false 9 Acked 1008 Read 2 Done Done Done false 10 Acked 1009 Read 3 Done Done Done false 11 Acked 1010 Read 5 Done Done Done false 12 Acked 1011 Read 9 Done Done Done false 13 Acked 1012 Write 3 Done Done Done false 14 Acked 1013 Read 3 Done Done Done false 15 Acked 1014 Read 10 Done Done Done false 16 Acked 1015 Read 3 Done Done Done false 17 Acked 1016 Read 7 Done Done Done false 18 Acked 1017 Read 5 Done Done Done false 19 Acked 1018 Write 7 Done Done Done false 20 Acked 1019 Write 10 Done Done Done false 21 Acked 1020 Write 9 Done Done Done false 22 Acked 1021 Write 3 Done Done Done false 23 Acked 1022 Write 7 Done Done Done false 24 Acked 1023 Write 9 Done Done Done false 25 Acked 1024 Read 1 Done Done Done false 26 Acked 1025 Write 8 Done Done Done false 27 Acked 1026 Read 6 Done Done Done false 28 Acked 1027 Write 5 Done Done Done false 29 Acked 1028 Write 10 Sent Sent Sent false 30 Acked 1029 Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:33:05.566786096Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1726} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 135 135 135 227 227 227 F F F 1 020-039 134 134 134 226 226 226 F F F 2 040-059 133 133 133 222 222 222 F F F 3 060-079 133 133 133 222 222 222 T T T 4 080-099 134 134 134 226 226 226 F F F 5 100-119 131 131 131 219 219 219 F F F 6 120-139 134 134 134 226 226 226 F F F 7 140-159 134 134 134 226 226 226 T T T 8 160-179 134 134 134 224 224 224 T T T 9 180-199 134 134 134 226 226 226 T T T 10 200-219 126 126 126 212 212 212 T T T 11 220-239 135 135 135 227 227 227 T T T 12 240-259 122 122 122 206 206 206 F F F 13 260-279 118 118 118 201 201 201 T T T 14 280-299 134 134 134 226 226 226 F F F 15 300-319 131 131 131 219 219 219 T T T 16 320-339 128 128 128 213 213 213 T T T 17 340-359 131 131 131 219 219 219 F F F 18 360-379 134 134 134 224 224 224 T T T 19 380-399 135 135 135 227 227 227 F F F 20 400-419 130 130 130 218 218 218 F F F 21 420-439 128 128 128 213 213 213 T T T 22 440-459 135 135 135 227 227 227 F F F 23 460-479 128 128 128 213 213 213 F F F 24 480-499 126 126 126 212 212 212 T T T 25 500-519 132 132 132 221 221 221 T T T 26 520-539 126 126 126 212 212 212 T T T 27 540-559 134 134 134 226 226 226 T T T 28 560-579 133 133 133 222 222 222 F F F 29 580-599 126 126 126 212 212 212 F F F Max gen: 135, Max flush: 227 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 135 135 135 227 227 227 F F F 1 020-039 134 134 134 226 226 226 F F F 2 040-059 133 133 133 222 222 222 F F F 3 060-079 136 136 136 228 228 228 F F F 4 080-099 134 134 134 226 226 226 F F F 5 100-119 131 131 131 219 219 219 F F F 6 120-139 134 134 134 226 226 226 F F F 7 140-159 136 136 136 228 228 228 F F F 8 160-179 136 136 136 228 228 228 F F F 9 180-199 136 136 136 228 228 228 F F F 10 200-219 136 136 136 228 228 228 F F F 11 220-239 136 136 136 228 228 228 F F F 12 240-259 122 122 122 206 206 206 F F F 13 260-279 136 136 136 228 228 228 F F F 14 280-299 134 134 134 226 226 226 F F F 15 300-319 136 136 136 228 228 228 F F F 16 320-339 136 136 136 228 228 228 F F F 17 340-359 131 131 131 219 219 219 F F F 18 360-379 136 136 136 228 228 228 F F F 19 380-399 135 135 135 227 227 227 F F F 20 400-419 130 130 130 218 218 218 F F F 21 420-439 136 136 136 228 228 228 F F F 22 440-459 135 135 135 227 227 227 F F F 23 460-479 128 128 128 213 213 213 F F F 24 480-499 136 136 136 228 228 228 F F F 25 500-519 136 136 136 228 228 228 F F F 26 520-539 136 136 136 228 228 228 F F F 27 540-559 136 136 136 228 228 228 F F F 28 560-579 133 133 133 222 222 222 F F F 29 580-599 126 126 126 212 212 212 F F F Max gen: 136, Max flush: 228 Begin loop 69 Downstairs to restart: 2 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.059894491Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.060216183Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.060244771Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.06026785Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734} {"msg":"Crucible b55e3de9-04ec-4623-be0a-08856c411b48 has session id: e3e79a54-01df-4a0a-aab0-500ff3bc7baf","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.060399212Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"Upstairs opts: Upstairs UUID: b55e3de9-04ec-4623-be0a-08856c411b48, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.060452319Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"Crucible stats registered with UUID: b55e3de9-04ec-4623-be0a-08856c411b48","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.060491117Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.091877309Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734} {"msg":"b55e3de9-04ec-4623-be0a-08856c411b48 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.091908428Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.091926237Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"client":"0","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.091953265Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"client":"0","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.091970764Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"client":"1","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.091991753Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"client":"1","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.092020812Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"client":"2","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.09205191Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"client":"2","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.092086018Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.092120516Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.092157524Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"ds_connection connected from Ok(127.0.0.1:57916)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.092193612Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"ds_connection connected from Ok(127.0.0.1:38522)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.09239433Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"ds_connection connected from Ok(127.0.0.1:46762)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.092416599Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.093014915Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"client":"1","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.093043254Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"client":"1","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.093058233Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"client":"0","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.093071812Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"client":"0","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.093094891Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"client":"2","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.093121929Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"client":"2","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.093701526Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"client":"1","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.093729985Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"client":"2","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.093746664Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"client":"0","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.094291033Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"client":"1","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.094316091Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"client":"1","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.094330561Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.094405237Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"client":"0","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.094424235Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"client":"0","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.094437564Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.094449944Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"client":"2","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.094463513Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"client":"2","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"[0]R flush_numbers[0..12]: [227, 226, 222, 228, 226, 219, 226, 228, 228, 228, 228, 228]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.094477542Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"":"downstairs","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"[0]R generation[0..12]: [135, 134, 133, 136, 134, 131, 134, 136, 136, 136, 136, 136]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.094491552Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"":"downstairs","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.094505551Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"":"downstairs","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"[1]R flush_numbers[0..12]: [227, 226, 222, 228, 226, 219, 226, 228, 228, 228, 228, 228]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.09451928Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"":"downstairs","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"[1]R generation[0..12]: [135, 134, 133, 136, 134, 131, 134, 136, 136, 136, 136, 136]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.094538339Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"":"downstairs","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.094553848Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"":"downstairs","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"[2]R flush_numbers[0..12]: [227, 226, 222, 228, 226, 219, 226, 228, 228, 228, 228, 228]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.094567617Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"":"downstairs","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"[2]R generation[0..12]: [135, 134, 133, 136, 134, 131, 134, 136, 136, 136, 136, 136]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.094581247Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"":"downstairs","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.094595276Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"":"downstairs","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"Max found gen is 137","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.094608205Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"":"downstairs","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"Generation requested: 138 >= found:137","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.094621464Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"":"downstairs","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"Next flush: 229","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.094634473Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"":"downstairs","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.094648023Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.094674181Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.09469724Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"client":"0","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.094722588Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"client":"1","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.094747787Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"client":"2","":"downstairs","session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.094764866Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.094777635Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"b55e3de9-04ec-4623-be0a-08856c411b48 is now active with session: e3e79a54-01df-4a0a-aab0-500ff3bc7baf","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-09-26T14:33:10.094793235Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.094810394Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734,"session_id":"e3e79a54-01df-4a0a-aab0-500ff3bc7baf"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:10.094822703Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 18 len 32768 data: 9 10 7 6 8 10 11 9 02/30 Read block 167 len 36864 03/30 Read block 181 len 20480 04/30 Read block 516 len 8192 05/30 Read block 292 len 40960 06/30 Write block 121 len 28672 data: 18 16 14 16 16 10 10 07/30 Read block 83 len 32768 08/30 Read block 525 len 32768 09/30 Read block 415 len 32768 10/30 Read block 425 len 12288 11/30 Write block 334 len 40960 data: 8 6 7 5 6 5 8 7 8 8 12/30 Write block 22 len 8192 data: 9 11 13/30 Read block 437 len 36864 14/30 Write block 352 len 40960 data: 14 16 14 15 14 15 15 14 15 13 15/30 Read block 375 len 8192 16/30 Write block 124 len 24576 data: 17 17 11 11 10 10 17/30 Write block 446 len 4096 data: 14 18/30 Write block 359 len 40960 data: 15 16 14 15 14 13 9 7 7 6 19/30 Read block 128 len 20480 20/30 Read block 80 len 16384 21/30 Flush 22/30 Write block 176 len 16384 data: 10 7 7 10 23/30 Flush 24/30 Write block 331 len 32768 data: 9 7 7 9 7 8 6 7 25/30 Read block 191 len 36864 26/30 Read block 401 len 36864 27/30 Read block 265 len 4096 28/30 Write block 442 len 16384 data: 11 12 14 15 29/30 Read block 527 len 32768 30/30 Read block 362 len 28672 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 8 Done Done Done false 25 Acked 1024 Read 9 Done Done Done false 26 Acked 1025 Read 9 Done Done Done false 27 Acked 1026 Read 1 Done Done Done false 28 Acked 1027 Write 4 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 7 7 6 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:33:10.120049364Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1734} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 229 229 229 F F F 1 020-039 138 138 138 229 229 229 F F F 2 040-059 133 133 133 222 222 222 F F F 3 060-079 136 136 136 228 228 228 F F F 4 080-099 134 134 134 226 226 226 F F F 5 100-119 131 131 131 219 219 219 F F F 6 120-139 138 138 138 229 229 229 F F F 7 140-159 136 136 136 228 228 228 F F F 8 160-179 138 138 138 230 230 230 F F F 9 180-199 136 136 136 228 228 228 F F F 10 200-219 136 136 136 228 228 228 F F F 11 220-239 136 136 136 228 228 228 F F F 12 240-259 122 122 122 206 206 206 F F F 13 260-279 136 136 136 228 228 228 F F F 14 280-299 134 134 134 226 226 226 F F F 15 300-319 136 136 136 228 228 228 F F F 16 320-339 138 138 138 229 229 229 T T T 17 340-359 138 138 138 229 229 229 F F F 18 360-379 138 138 138 229 229 229 F F F 19 380-399 135 135 135 227 227 227 F F F 20 400-419 130 130 130 218 218 218 F F F 21 420-439 136 136 136 228 228 228 F F F 22 440-459 138 138 138 229 229 229 T T T 23 460-479 128 128 128 213 213 213 F F F 24 480-499 136 136 136 228 228 228 F F F 25 500-519 136 136 136 228 228 228 F F F 26 520-539 136 136 136 228 228 228 F F F 27 540-559 136 136 136 228 228 228 F F F 28 560-579 133 133 133 222 222 222 F F F 29 580-599 126 126 126 212 212 212 F F F Max gen: 138, Max flush: 230 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 229 229 229 F F F 1 020-039 138 138 138 229 229 229 F F F 2 040-059 133 133 133 222 222 222 F F F 3 060-079 136 136 136 228 228 228 F F F 4 080-099 134 134 134 226 226 226 F F F 5 100-119 131 131 131 219 219 219 F F F 6 120-139 138 138 138 229 229 229 F F F 7 140-159 136 136 136 228 228 228 F F F 8 160-179 138 138 138 230 230 230 F F F 9 180-199 136 136 136 228 228 228 F F F 10 200-219 136 136 136 228 228 228 F F F 11 220-239 136 136 136 228 228 228 F F F 12 240-259 122 122 122 206 206 206 F F F 13 260-279 136 136 136 228 228 228 F F F 14 280-299 134 134 134 226 226 226 F F F 15 300-319 136 136 136 228 228 228 F F F 16 320-339 139 139 139 231 231 231 F F F 17 340-359 138 138 138 229 229 229 F F F 18 360-379 138 138 138 229 229 229 F F F 19 380-399 135 135 135 227 227 227 F F F 20 400-419 130 130 130 218 218 218 F F F 21 420-439 136 136 136 228 228 228 F F F 22 440-459 139 139 139 231 231 231 F F F 23 460-479 128 128 128 213 213 213 F F F 24 480-499 136 136 136 228 228 228 F F F 25 500-519 136 136 136 228 228 228 F F F 26 520-539 136 136 136 228 228 228 F F F 27 540-559 136 136 136 228 228 228 F F F 28 560-579 133 133 133 222 222 222 F F F 29 580-599 126 126 126 212 212 212 F F F Max gen: 139, Max flush: 231 Begin loop 70 Downstairs to restart: 2 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.434868388Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.435173341Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.435201479Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.435218858Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742} {"msg":"Crucible 190d6a1b-36d9-4547-9269-b4807d60fb1d has session id: bc498e45-4e32-47fe-9916-ee52cf3800dc","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.435385689Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"Upstairs opts: Upstairs UUID: 190d6a1b-36d9-4547-9269-b4807d60fb1d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.435434176Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"Crucible stats registered with UUID: 190d6a1b-36d9-4547-9269-b4807d60fb1d","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.435456785Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.466549254Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742} {"msg":"190d6a1b-36d9-4547-9269-b4807d60fb1d active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.466579992Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.466598071Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"client":"0","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.46662414Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"client":"0","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.466643418Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"client":"1","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.466660597Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"client":"1","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.466676926Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"client":"2","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.466693456Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"client":"2","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.466710175Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.466727403Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.466752992Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:63480)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.46678602Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:36443)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.466817918Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:61522)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.466866766Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.467600654Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"client":"0","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.467628463Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"client":"0","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.467762665Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"client":"1","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.467791653Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"client":"1","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.467807402Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"client":"2","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.467826931Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"client":"2","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.4682055Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"client":"0","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.468235478Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"client":"1","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.468496823Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"client":"2","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.468825405Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"client":"1","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.468852773Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"client":"1","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.468871962Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.468885331Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"client":"0","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.468898891Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"client":"0","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.4689122Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.469016974Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"client":"2","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.469036263Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"client":"2","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"[0]R flush_numbers[0..12]: [229, 229, 222, 228, 226, 219, 229, 228, 230, 228, 228, 228]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.469050592Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"":"downstairs","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"[0]R generation[0..12]: [138, 138, 133, 136, 134, 131, 138, 136, 138, 136, 136, 136]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.469064841Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"":"downstairs","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.46907929Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"":"downstairs","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"[1]R flush_numbers[0..12]: [229, 229, 222, 228, 226, 219, 229, 228, 230, 228, 228, 228]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.469093489Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"":"downstairs","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"[1]R generation[0..12]: [138, 138, 133, 136, 134, 131, 138, 136, 138, 136, 136, 136]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.469107269Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"":"downstairs","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.469121198Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"":"downstairs","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"[2]R flush_numbers[0..12]: [229, 229, 222, 228, 226, 219, 229, 228, 230, 228, 228, 228]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.469134957Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"":"downstairs","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"[2]R generation[0..12]: [138, 138, 133, 136, 134, 131, 138, 136, 138, 136, 136, 136]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.469148737Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"":"downstairs","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.469165356Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"":"downstairs","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.469178495Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"":"downstairs","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.469191604Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"":"downstairs","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"Next flush: 232","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.469204393Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"":"downstairs","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.469217502Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"":"downstairs","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-26T14:33:14.469230382Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.469245351Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"client":"0","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.46925851Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"client":"1","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.46927171Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"client":"2","":"downstairs","session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.469284829Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.469297128Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"190d6a1b-36d9-4547-9269-b4807d60fb1d is now active with session: bc498e45-4e32-47fe-9916-ee52cf3800dc","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.469309797Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.469322367Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742,"session_id":"bc498e45-4e32-47fe-9916-ee52cf3800dc"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:14.469334686Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 412 len 32768 data: 7 10 11 10 11 10 10 9 02/30 Read block 262 len 40960 03/30 Write block 314 len 40960 data: 16 15 17 17 11 13 13 11 8 8 04/30 Read block 298 len 40960 05/30 Read block 535 len 24576 06/30 Write block 460 len 20480 data: 7 8 5 7 7 07/30 Write block 153 len 32768 data: 9 9 11 15 14 12 12 9 08/30 Read block 302 len 32768 09/30 Write block 216 len 24576 data: 15 16 13 12 11 10 10/30 Flush 11/30 Write block 170 len 8192 data: 12 13 12/30 Write block 275 len 24576 data: 14 15 14 15 14 13 13/30 Read block 159 len 8192 14/30 Read block 247 len 8192 15/30 Read block 70 len 28672 16/30 Read block 336 len 20480 17/30 Write block 308 len 32768 data: 13 11 7 9 10 14 17 16 18/30 Write block 285 len 32768 data: 17 15 14 14 16 15 14 12 19/30 Write block 267 len 8192 data: 12 13 20/30 Write block 206 len 4096 data: 11 21/30 Read block 437 len 24576 22/30 Write block 471 len 36864 data: 9 9 9 8 8 10 8 10 13 23/30 Write block 81 len 20480 data: 14 13 10 11 11 24/30 Write block 503 len 28672 data: 15 16 14 12 12 10 8 25/30 Flush 26/30 Write block 188 len 24576 data: 11 9 9 8 10 8 27/30 Read block 258 len 32768 28/30 Read block 49 len 12288 29/30 Read block 166 len 8192 30/30 Write block 577 len 12288 data: 16 14 10 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 6 Done Done Done false 27 Acked 1026 Read 8 Done Done Done false 28 Acked 1027 Read 3 Done Done Done false 29 Acked 1028 Read 2 Done Done Done false 30 Acked 1029 Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:33:14.494062366Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1742} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 229 229 229 F F F 1 020-039 138 138 138 229 229 229 F F F 2 040-059 133 133 133 222 222 222 F F F 3 060-079 136 136 136 228 228 228 F F F 4 080-099 140 140 140 233 233 233 F F F 5 100-119 131 131 131 219 219 219 F F F 6 120-139 138 138 138 229 229 229 F F F 7 140-159 140 140 140 232 232 232 F F F 8 160-179 140 140 140 233 233 233 F F F 9 180-199 136 136 136 228 228 228 T T T 10 200-219 140 140 140 233 233 233 F F F 11 220-239 140 140 140 232 232 232 F F F 12 240-259 122 122 122 206 206 206 F F F 13 260-279 140 140 140 233 233 233 F F F 14 280-299 140 140 140 233 233 233 F F F 15 300-319 140 140 140 233 233 233 F F F 16 320-339 140 140 140 232 232 232 F F F 17 340-359 138 138 138 229 229 229 F F F 18 360-379 138 138 138 229 229 229 F F F 19 380-399 135 135 135 227 227 227 F F F 20 400-419 140 140 140 232 232 232 F F F 21 420-439 136 136 136 228 228 228 F F F 22 440-459 139 139 139 231 231 231 F F F 23 460-479 140 140 140 233 233 233 F F F 24 480-499 136 136 136 228 228 228 F F F 25 500-519 140 140 140 233 233 233 F F F 26 520-539 136 136 136 228 228 228 F F F 27 540-559 136 136 136 228 228 228 F F F 28 560-579 133 133 133 222 222 222 T T T 29 580-599 126 126 126 212 212 212 F F F Max gen: 140, Max flush: 233 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 229 229 229 F F F 1 020-039 138 138 138 229 229 229 F F F 2 040-059 133 133 133 222 222 222 F F F 3 060-079 136 136 136 228 228 228 F F F 4 080-099 140 140 140 233 233 233 F F F 5 100-119 131 131 131 219 219 219 F F F 6 120-139 138 138 138 229 229 229 F F F 7 140-159 140 140 140 232 232 232 F F F 8 160-179 140 140 140 233 233 233 F F F 9 180-199 141 141 141 234 234 234 F F F 10 200-219 140 140 140 233 233 233 F F F 11 220-239 140 140 140 232 232 232 F F F 12 240-259 122 122 122 206 206 206 F F F 13 260-279 140 140 140 233 233 233 F F F 14 280-299 140 140 140 233 233 233 F F F 15 300-319 140 140 140 233 233 233 F F F 16 320-339 140 140 140 232 232 232 F F F 17 340-359 138 138 138 229 229 229 F F F 18 360-379 138 138 138 229 229 229 F F F 19 380-399 135 135 135 227 227 227 F F F 20 400-419 140 140 140 232 232 232 F F F 21 420-439 136 136 136 228 228 228 F F F 22 440-459 139 139 139 231 231 231 F F F 23 460-479 140 140 140 233 233 233 F F F 24 480-499 136 136 136 228 228 228 F F F 25 500-519 140 140 140 233 233 233 F F F 26 520-539 136 136 136 228 228 228 F F F 27 540-559 136 136 136 228 228 228 F F F 28 560-579 141 141 141 234 234 234 F F F 29 580-599 126 126 126 212 212 212 F F F Max gen: 141, Max flush: 234 Begin loop 71 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.804133148Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.804385734Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.804410713Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.804437291Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750} {"msg":"Crucible 5cb5c998-9dd5-4f52-8620-04e1e17085e6 has session id: cae792d9-1baf-424b-a36b-390f8ba6caa3","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.804597562Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"Upstairs opts: Upstairs UUID: 5cb5c998-9dd5-4f52-8620-04e1e17085e6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.804642349Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"Crucible stats registered with UUID: 5cb5c998-9dd5-4f52-8620-04e1e17085e6","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.804664568Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.835640924Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750} {"msg":"5cb5c998-9dd5-4f52-8620-04e1e17085e6 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.835671242Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.835690141Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"client":"0","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.835717109Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"client":"0","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.835734919Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"client":"1","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.835754367Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"client":"1","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.835795615Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"client":"2","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.835854242Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"client":"2","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.83588749Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.835922898Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.835953136Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"ds_connection connected from Ok(127.0.0.1:49525)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.83606491Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"ds_connection connected from Ok(127.0.0.1:58294)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.836157825Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"ds_connection connected from Ok(127.0.0.1:46818)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.83624761Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.836815507Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"client":"0","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.836846016Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"client":"0","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.836861255Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"client":"2","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.836922641Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"client":"2","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.83695003Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"client":"1","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.836964949Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"client":"1","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.837534387Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"client":"1","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.837562855Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"client":"2","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.837637461Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"client":"0","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.83817652Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"client":"2","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.838201199Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"client":"2","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.838217488Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.838287394Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"client":"1","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.838307723Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"client":"1","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.838321602Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.838333771Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"client":"0","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.83835052Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"client":"0","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"[0]R flush_numbers[0..12]: [229, 229, 222, 228, 233, 219, 229, 232, 233, 234, 233, 232]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.838371529Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"":"downstairs","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"[0]R generation[0..12]: [138, 138, 133, 136, 140, 131, 138, 140, 140, 141, 140, 140]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.838385989Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"":"downstairs","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.838400098Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"":"downstairs","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"[1]R flush_numbers[0..12]: [229, 229, 222, 228, 233, 219, 229, 232, 233, 234, 233, 232]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.838414257Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"":"downstairs","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"[1]R generation[0..12]: [138, 138, 133, 136, 140, 131, 138, 140, 140, 141, 140, 140]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.838428696Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"":"downstairs","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.838442855Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"":"downstairs","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"[2]R flush_numbers[0..12]: [229, 229, 222, 228, 233, 219, 229, 232, 233, 234, 233, 232]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.838456534Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"":"downstairs","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"[2]R generation[0..12]: [138, 138, 133, 136, 140, 131, 138, 140, 140, 141, 140, 140]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.838470174Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"":"downstairs","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.838483973Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"":"downstairs","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.838496952Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"":"downstairs","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.838510291Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"":"downstairs","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"Next flush: 235","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.838523211Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"":"downstairs","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.83853637Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"":"downstairs","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.838553199Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.838565808Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"client":"0","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.838579167Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"client":"1","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {Wait for a query_work_queue command to finish before sending IO "msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.838596237Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"client":"2","":"downstairs","session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.838610196Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.838622685Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"5cb5c998-9dd5-4f52-8620-04e1e17085e6 is now active with session: cae792d9-1baf-424b-a36b-390f8ba6caa3","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.838635554Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.838647474Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750,"session_id":"cae792d9-1baf-424b-a36b-390f8ba6caa3"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:18.838660853Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 587 len 16384 data: 14 13 12 11 02/30 Read block 117 len 12288 03/30 Read block 478 len 40960 04/30 Read block 39 len 12288 05/30 Write block 83 len 4096 data: 11 06/30 Write block 196 len 20480 data: 10 9 11 7 9 07/30 Write block 373 len 32768 data: 8 10 10 8 9 11 8 9 08/30 Read block 418 len 4096 09/30 Read block 243 len 40960 10/30 Read block 497 len 4096 11/30 Read block 464 len 12288 12/30 Read block 490 len 24576 13/30 Write block 200 len 8192 data: 10 8 14/30 Write block 321 len 32768 data: 12 9 9 9 11 10 12 13 15/30 Read block 76 len 16384 16/30 Read block 248 len 20480 17/30 Write block 594 len 24576 data: 7 6 5 6 4 3 18/30 Read block 191 len 32768 19/30 Read block 59 len 32768 20/30 Read block 452 len 32768 21/30 Read block 269 len 12288 22/30 Write block 128 len 32768 data: 11 11 10 10 10 10 9 7 23/30 Write block 570 len 16384 data: 13 16 17 17 24/30 Read block 453 len 20480 25/30 Read block 255 len 8192 26/30 Write block 15 len 12288 data: 10 9 9 27/30 Read block 450 len 40960 28/30 Write block 140 len 12288 data: 10 11 8 29/30 Read block 239 len 36864 30/30 Write block 313 len 12288 data: 15 18 17 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Write 4 Done Done Done false 2 Acked 1001 Read 3 Done Done Done false 3 Acked 1002 Read 10 Done Done Done false 4 Acked 1003 Read 3 Done Done Done false 5 Acked 1004 Write 1 Done Done Done false 6 Acked 1005 Write 5 Done Done Done false 7 Acked 1006 Write 8 Done Done Done false 8 Acked 1007 Read 1 Done Done Done false 9 Acked 1008 Read 10 Done Done Done false 10 Acked 1009 Read 1 Done Done Done false 11 Acked 1010 Read 3 Done Done Done false 12 Acked 1011 Read 6 Done Done Done false 13 Acked 1012 Write 2 Done Done Done false 14 Acked 1013 Write 8 Done Done Done false 15 Acked 1014 Read 4 Done Done Done false 16 Acked 1015 Read 5 Done Done Done false 17 Acked 1016 Write 6 Done Done Done false 18 Acked 1017 Read 8 Done Done Done false 19 Acked 1018 Read 8 Done Done Done false 20 Acked 1019 Read 8 Done Done Done false 21 Acked 1020 Read 3 Done Done Done false 22 Acked 1021 Write 8 Done Done Done false 23 Acked 1022 Write 4 Done Done Done false 24 Acked 1023 Read 5 Done Done Done false 25 Acked 1024 Read 2 Done Done Done false 26 Acked 1025 Write 3 Done Done Done false 27 Acked 1026 Read 10 Done Done Done false 28 Acked 1027 Write 3 Done Done Done false 29 Acked 1028 Read 9 Done Done Done false 30 Acked 1029 Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:33:18.858094982Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1750} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 229 229 229 T T T 1 020-039 138 138 138 229 229 229 F F F 2 040-059 133 133 133 222 222 222 F F F 3 060-079 136 136 136 228 228 228 F F F 4 080-099 140 140 140 233 233 233 T T T 5 100-119 131 131 131 219 219 219 F F F 6 120-139 138 138 138 229 229 229 T T T 7 140-159 140 140 140 232 232 232 T T T 8 160-179 140 140 140 233 233 233 F F F 9 180-199 141 141 141 234 234 234 T T T 10 200-219 140 140 140 233 233 233 T T T 11 220-239 140 140 140 232 232 232 F F F 12 240-259 122 122 122 206 206 206 F F F 13 260-279 140 140 140 233 233 233 F F F 14 280-299 140 140 140 233 233 233 F F F 15 300-319 140 140 140 233 233 233 T T T 16 320-339 140 140 140 232 232 232 T T T 17 340-359 138 138 138 229 229 229 F F F 18 360-379 138 138 138 229 229 229 T T T 19 380-399 135 135 135 227 227 227 T T T 20 400-419 140 140 140 232 232 232 F F F 21 420-439 136 136 136 228 228 228 F F F 22 440-459 139 139 139 231 231 231 F F F 23 460-479 140 140 140 233 233 233 F F F 24 480-499 136 136 136 228 228 228 F F F 25 500-519 140 140 140 233 233 233 F F F 26 520-539 136 136 136 228 228 228 F F F 27 540-559 136 136 136 228 228 228 F F F 28 560-579 141 141 141 234 234 234 T T T 29 580-599 126 126 126 212 212 212 T T T Max gen: 141, Max flush: 234 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 235 235 235 F F F 1 020-039 138 138 138 229 229 229 F F F 2 040-059 133 133 133 222 222 222 F F F 3 060-079 136 136 136 228 228 228 F F F 4 080-099 142 142 142 235 235 235 F F F 5 100-119 131 131 131 219 219 219 F F F 6 120-139 142 142 142 235 235 235 F F F 7 140-159 142 142 142 235 235 235 F F F 8 160-179 140 140 140 233 233 233 F F F 9 180-199 142 142 142 235 235 235 F F F 10 200-219 142 142 142 235 235 235 F F F 11 220-239 140 140 140 232 232 232 F F F 12 240-259 122 122 122 206 206 206 F F F 13 260-279 140 140 140 233 233 233 F F F 14 280-299 140 140 140 233 233 233 F F F 15 300-319 142 142 142 235 235 235 F F F 16 320-339 142 142 142 235 235 235 F F F 17 340-359 138 138 138 229 229 229 F F F 18 360-379 142 142 142 235 235 235 F F F 19 380-399 142 142 142 235 235 235 F F F 20 400-419 140 140 140 232 232 232 F F F 21 420-439 136 136 136 228 228 228 F F F 22 440-459 139 139 139 231 231 231 F F F 23 460-479 140 140 140 233 233 233 F F F 24 480-499 136 136 136 228 228 228 F F F 25 500-519 140 140 140 233 233 233 F F F 26 520-539 136 136 136 228 228 228 F F F 27 540-559 136 136 136 228 228 228 F F F 28 560-579 142 142 142 235 235 235 F F F 29 580-599 142 142 142 235 235 235 F F F Max gen: 142, Max flush: 235 Begin loop 72 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.33157618Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.331833256Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.331851775Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.331866814Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758} {"msg":"Crucible 399f7c05-33e3-4708-93ca-9bb57d7a3483 has session id: 2ddde767-8ac7-4773-b4bf-02761b5b4b1e","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.332077032Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"Upstairs opts: Upstairs UUID: 399f7c05-33e3-4708-93ca-9bb57d7a3483, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.332122769Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"Crucible stats registered with UUID: 399f7c05-33e3-4708-93ca-9bb57d7a3483","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.332153608Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.363052608Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758} {"msg":"399f7c05-33e3-4708-93ca-9bb57d7a3483 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.363087306Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.363112404Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"client":"0","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.363158352Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"client":"0","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.36319607Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"client":"1","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.363229178Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"client":"1","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.363261026Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"client":"2","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.363294404Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"client":"2","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.363321292Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.363352591Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.363383399Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"ds_connection connected from Ok(127.0.0.1:53418)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.363526991Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"ds_connection connected from Ok(127.0.0.1:57308)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.363576538Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"ds_connection connected from Ok(127.0.0.1:45143)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.363598687Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.364186993Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"client":"1","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.364224441Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"client":"1","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.364255599Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"client":"2","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.364284428Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"client":"2","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.364310846Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"client":"0","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.364326906Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"client":"0","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.364859705Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"client":"2","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.365031215Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"client":"1","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.365073153Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"client":"0","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.365355697Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"client":"2","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.365381926Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"client":"2","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.365396335Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.365564885Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"client":"1","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.365583594Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"client":"1","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.365596914Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.365639731Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"client":"0","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.36565677Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"client":"0","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"[0]R flush_numbers[0..12]: [235, 229, 222, 228, 235, 219, 235, 235, 233, 235, 235, 232]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.365676599Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"":"downstairs","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"[0]R generation[0..12]: [142, 138, 133, 136, 142, 131, 142, 142, 140, 142, 142, 140]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.365690748Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"":"downstairs","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.365704748Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"":"downstairs","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"[1]R flush_numbers[0..12]: [235, 229, 222, 228, 235, 219, 235, 235, 233, 235, 235, 232]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.365718497Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"":"downstairs","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"[1]R generation[0..12]: [142, 138, 133, 136, 142, 131, 142, 142, 140, 142, 142, 140]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.365732106Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"":"downstairs","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.365745935Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"":"downstairs","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"[2]R flush_numbers[0..12]: [235, 229, 222, 228, 235, 219, 235, 235, 233, 235, 235, 232]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.365759354Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"":"downstairs","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"[2]R generation[0..12]: [142, 138, 133, 136, 142, 131, 142, 142, 140, 142, 142, 140]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.365772764Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"":"downstairs","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.365786633Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"":"downstairs","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"Max found gen is 143","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.365799422Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"":"downstairs","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"Generation requested: 144 >= found:143","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.365812441Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"":"downstairs","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"Next flush: 236","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.36582559Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"":"downstairs","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.3658421Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"":"downstairs","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-26T14:33:23.365856069Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.365869208Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"client":"0","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.365882607Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"client":"1","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.365895647Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"client":"2","":"downstairs","session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.365909696Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.365929035Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"399f7c05-33e3-4708-93ca-9bb57d7a3483 is now active with session: 2ddde767-8ac7-4773-b4bf-02761b5b4b1e","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.365956163Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.365968842Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758,"session_id":"2ddde767-8ac7-4773-b4bf-02761b5b4b1e"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:23.365981212Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 297 len 16384 data: 8 13 11 10 02/30 Read block 152 len 40960 03/30 Write block 539 len 20480 data: 13 13 12 11 12 04/30 Write block 188 len 20480 data: 12 10 10 9 11 05/30 Write block 69 len 24576 data: 9 10 12 13 12 12 06/30 Read block 551 len 28672 07/30 Read block 286 len 12288 08/30 Write block 166 len 40960 data: 6 10 9 11 13 14 13 12 10 9 09/30 Read block 525 len 8192 10/30 Write block 156 len 24576 data: 16 15 13 13 10 8 11/30 Read block 218 len 28672 12/30 Read block 165 len 12288 13/30 Write block 8 len 20480 data: 6 8 10 9 9 14/30 Flush 15/30 Write block 187 len 32768 data: 10 13 11 11 10 12 9 8 16/30 Read block 386 len 40960 17/30 Flush 18/30 Write block 440 len 20480 data: 11 11 12 13 15 19/30 Write block 508 len 36864 data: 11 9 10 9 10 8 9 7 7 20/30 Write block 150 len 20480 data: 6 9 9 10 10 21/30 Read block 353 len 24576 22/30 Flush 23/30 Write block 542 len 28672 data: 12 13 12 16 17 17 18 24/30 Read block 513 len 32768 25/30 Read block 41 len 8192 26/30 Read block 378 len 24576 27/30 Read block 249 len 20480 28/30 Read block 36 len 28672 29/30 Read block 296 len 4096 30/30 Write block 527 len 28672 data: 12 12 10 10 12 13 14 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 7 Done Done Done false 24 Acked 1023 Read 8 Done Done Done false 25 Acked 1024 Read 2 Done Done Done false 26 Acked 1025 Read 6 Done Done Done false 27 Acked 1026 Read 5 Done Done Done false 28 Acked 1027 Read 7 Done Done Done false 29 Acked 1028 Read 1 Done Done Done false 30 Acked 1029 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:33:23.391881785Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1758} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 236 236 236 F F F 1 020-039 138 138 138 229 229 229 F F F 2 040-059 133 133 133 222 222 222 F F F 3 060-079 144 144 144 236 236 236 F F F 4 080-099 142 142 142 235 235 235 F F F 5 100-119 131 131 131 219 219 219 F F F 6 120-139 142 142 142 235 235 235 F F F 7 140-159 144 144 144 238 238 238 F F F 8 160-179 144 144 144 236 236 236 F F F 9 180-199 144 144 144 237 237 237 F F F 10 200-219 142 142 142 235 235 235 F F F 11 220-239 140 140 140 232 232 232 F F F 12 240-259 122 122 122 206 206 206 F F F 13 260-279 140 140 140 233 233 233 F F F 14 280-299 144 144 144 236 236 236 F F F 15 300-319 144 144 144 236 236 236 F F F 16 320-339 142 142 142 235 235 235 F F F 17 340-359 138 138 138 229 229 229 F F F 18 360-379 142 142 142 235 235 235 F F F 19 380-399 142 142 142 235 235 235 F F F 20 400-419 140 140 140 232 232 232 F F F 21 420-439 136 136 136 228 228 228 F F F 22 440-459 144 144 144 238 238 238 F F F 23 460-479 140 140 140 233 233 233 F F F 24 480-499 136 136 136 228 228 228 F F F 25 500-519 144 144 144 238 238 238 F F F 26 520-539 144 144 144 236 236 236 T T T 27 540-559 144 144 144 236 236 236 T T T 28 560-579 142 142 142 235 235 235 F F F 29 580-599 142 142 142 235 235 235 F F F Max gen: 144, Max flush: 238 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 236 236 236 F F F 1 020-039 138 138 138 229 229 229 F F F 2 040-059 133 133 133 222 222 222 F F F 3 060-079 144 144 144 236 236 236 F F F 4 080-099 142 142 142 235 235 235 F F F 5 100-119 131 131 131 219 219 219 F F F 6 120-139 142 142 142 235 235 235 F F F 7 140-159 144 144 144 238 238 238 F F F 8 160-179 144 144 144 236 236 236 F F F 9 180-199 144 144 144 237 237 237 F F F 10 200-219 142 142 142 235 235 235 F F F 11 220-239 140 140 140 232 232 232 F F F 12 240-259 122 122 122 206 206 206 F F F 13 260-279 140 140 140 233 233 233 F F F 14 280-299 144 144 144 236 236 236 F F F 15 300-319 144 144 144 236 236 236 F F F 16 320-339 142 142 142 235 235 235 F F F 17 340-359 138 138 138 229 229 229 F F F 18 360-379 142 142 142 235 235 235 F F F 19 380-399 142 142 142 235 235 235 F F F 20 400-419 140 140 140 232 232 232 F F F 21 420-439 136 136 136 228 228 228 F F F 22 440-459 144 144 144 238 238 238 F F F 23 460-479 140 140 140 233 233 233 F F F 24 480-499 136 136 136 228 228 228 F F F 25 500-519 144 144 144 238 238 238 F F F 26 520-539 145 145 145 239 239 239 F F F 27 540-559 145 145 145 239 239 239 F F F 28 560-579 142 142 142 235 235 235 F F F 29 580-599 142 142 142 235 235 235 F F F Max gen: 145, Max flush: 239 Begin loop 73 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.763974333Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.764331463Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.764366161Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.764399379Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766} {"msg":"Crucible e4c8953c-ef4a-45e7-837c-4bc20c335bf6 has session id: bccf8624-de6b-4a38-ac96-6210696eb66c","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.764425958Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"Upstairs opts: Upstairs UUID: e4c8953c-ef4a-45e7-837c-4bc20c335bf6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.764473915Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"Crucible stats registered with UUID: e4c8953c-ef4a-45e7-837c-4bc20c335bf6","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.764501753Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.795924924Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766} {"msg":"e4c8953c-ef4a-45e7-837c-4bc20c335bf6 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.796006919Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.796033647Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"client":"0","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.796061436Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"client":"0","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.796079285Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"client":"1","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.796096504Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"client":"1","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.796113013Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"client":"2","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.796129792Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"client":"2","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.796145861Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.79616809Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.796205857Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"ds_connection connected from Ok(127.0.0.1:36137)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.796238446Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"ds_connection connected from Ok(127.0.0.1:51514)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.796272444Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"ds_connection connected from Ok(127.0.0.1:38955)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.796497091Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.797034221Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"client":"1","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.797068709Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"client":"1","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.797096107Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"client":"2","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.797119326Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"client":"2","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.797134185Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"client":"0","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.797147974Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"client":"0","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.797681784Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"client":"1","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.797800597Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"client":"2","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.797839805Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"client":"0","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.79828112Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"client":"1","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.798310948Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"client":"1","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.798328107Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.7984573Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"client":"2","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.798482879Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"client":"2","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.798497148Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.798509777Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"client":"0","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.798523126Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"client":"0","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"[0]R flush_numbers[0..12]: [236, 229, 222, 236, 235, 219, 235, 238, 236, 237, 235, 232]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.798537396Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"":"downstairs","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"[0]R generation[0..12]: [144, 138, 133, 144, 142, 131, 142, 144, 144, 144, 142, 140]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.798551935Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"":"downstairs","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.798570064Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"":"downstairs","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"[1]R flush_numbers[0..12]: [236, 229, 222, 236, 235, 219, 235, 238, 236, 237, 235, 232]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.798584283Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"":"downstairs","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"[1]R generation[0..12]: [144, 138, 133, 144, 142, 131, 142, 144, 144, 144, 142, 140]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.798598192Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"":"downstairs","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.798612201Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"":"downstairs","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"[2]R flush_numbers[0..12]: [236, 229, 222, 236, 235, 219, 235, 238, 236, 237, 235, 232]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.79862594Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"":"downstairs","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"[2]R generation[0..12]: [144, 138, 133, 144, 142, 131, 142, 144, 144, 144, 142, 140]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.79863997Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"":"downstairs","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.798654019Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"":"downstairs","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.798667118Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"":"downstairs","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.798680517Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"":"downstairs","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"Next flush: 240","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.798693517Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"":"downstairs","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.798713166Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"":"downstairs","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.798732474Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.798755173Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"client":"0","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.798771342Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"client":"1","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.798787141Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"client":"2","":"downstairs","session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.798800661Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.79881843Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"e4c8953c-ef4a-45e7-837c-4bc20c335bf6 is now active with session: bccf8624-de6b-4a38-ac96-6210696eb66c","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.798841908Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.798855747Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766,"session_id":"bccf8624-de6b-4a38-ac96-6210696eb66c"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:27.798868257Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 316 len 20480 data: 18 18 12 14 14 02/30 Write block 283 len 12288 data: 11 12 18 03/30 Write block 231 len 16384 data: 3 3 5 4 04/30 Read block 97 len 32768 05/30 Write block 541 len 28672 data: 13 13 14 13 17 18 18 06/30 Flush 07/30 Write block 50 len 16384 data: 15 13 14 12 08/30 Write block 539 len 20480 data: 14 14 14 14 15 09/30 Write block 185 len 20480 data: 13 9 11 14 12 10/30 Write block 330 len 32768 data: 9 10 8 8 10 8 9 7 11/30 Read block 227 len 8192 12/30 Write block 90 len 36864 data: 15 14 13 13 11 9 8 9 8 13/30 Read block 311 len 8192 14/30 Write block 590 len 32768 data: 12 8 7 7 8 7 6 7 15/30 Write block 210 len 8192 data: 11 10 16/30 Read block 136 len 28672 17/30 Write block 481 len 24576 data: 12 11 10 9 9 9 18/30 Read block 180 len 12288 19/30 Read block 94 len 24576 20/30 Write block 58 len 24576 data: 10 10 10 9 7 9 21/30 Write block 175 len 4096 data: 10 22/30 Flush 23/30 Write block 330 len 28672 data: 10 11 9 9 11 9 10 24/30 Read block 443 len 36864 25/30 Read block 525 len 32768 26/30 Write block 454 len 4096 data: 9 27/30 Write block 316 len 40960 data: 19 19 13 15 15 13 10 10 10 12 28/30 Read block 578 len 16384 29/30 Write block 134 len 40960 data: 10 8 9 9 7 8 11 12 9 10 30/30 Read block 374 len 12288 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 7 Done Done Done false 24 Acked 1023 Read 9 Done Done Done false 25 Acked 1024 Read 8 Done Done Done false 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Write 10 Done Done Done false 28 Acked 1027 Read 4 Done Done Done false 29 Acked 1028 Write 10 Done Done Sent false 30 Acked 1029 Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 8 7 6 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:33:27.824347864Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1766} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 236 236 236 F F F 1 020-039 138 138 138 229 229 229 F F F 2 040-059 146 146 146 241 241 241 F F F 3 060-079 146 146 146 241 241 241 F F F 4 080-099 146 146 146 241 241 241 F F F 5 100-119 131 131 131 219 219 219 F F F 6 120-139 142 142 142 235 235 235 T T T 7 140-159 144 144 144 238 238 238 T T T 8 160-179 146 146 146 241 241 241 F F F 9 180-199 146 146 146 241 241 241 F F F 10 200-219 146 146 146 241 241 241 F F F 11 220-239 146 146 146 240 240 240 F F F 12 240-259 122 122 122 206 206 206 F F F 13 260-279 140 140 140 233 233 233 F F F 14 280-299 146 146 146 240 240 240 F F F 15 300-319 146 146 146 240 240 240 T T T 16 320-339 146 146 146 241 241 241 T T T 17 340-359 138 138 138 229 229 229 F F F 18 360-379 142 142 142 235 235 235 F F F 19 380-399 142 142 142 235 235 235 F F F 20 400-419 140 140 140 232 232 232 F F F 21 420-439 136 136 136 228 228 228 F F F 22 440-459 144 144 144 238 238 238 T T T 23 460-479 140 140 140 233 233 233 F F F 24 480-499 146 146 146 241 241 241 F F F 25 500-519 144 144 144 238 238 238 F F F 26 520-539 146 146 146 241 241 241 F F F 27 540-559 146 146 146 241 241 241 F F F 28 560-579 142 142 142 235 235 235 F F F 29 580-599 146 146 146 241 241 241 F F F Max gen: 146, Max flush: 241 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 236 236 236 F F F 1 020-039 138 138 138 229 229 229 F F F 2 040-059 146 146 146 241 241 241 F F F 3 060-079 146 146 146 241 241 241 F F F 4 080-099 146 146 146 241 241 241 F F F 5 100-119 131 131 131 219 219 219 F F F 6 120-139 147 147 147 242 242 242 F F F 7 140-159 147 147 147 242 242 242 F F F 8 160-179 146 146 146 241 241 241 F F F 9 180-199 146 146 146 241 241 241 F F F 10 200-219 146 146 146 241 241 241 F F F 11 220-239 146 146 146 240 240 240 F F F 12 240-259 122 122 122 206 206 206 F F F 13 260-279 140 140 140 233 233 233 F F F 14 280-299 146 146 146 240 240 240 F F F 15 300-319 147 147 147 242 242 242 F F F 16 320-339 147 147 147 242 242 242 F F F 17 340-359 138 138 138 229 229 229 F F F 18 360-379 142 142 142 235 235 235 F F F 19 380-399 142 142 142 235 235 235 F F F 20 400-419 140 140 140 232 232 232 F F F 21 420-439 136 136 136 228 228 228 F F F 22 440-459 147 147 147 242 242 242 F F F 23 460-479 140 140 140 233 233 233 F F F 24 480-499 146 146 146 241 241 241 F F F 25 500-519 144 144 144 238 238 238 F F F 26 520-539 146 146 146 241 241 241 F F F 27 540-559 146 146 146 241 241 241 F F F 28 560-579 142 142 142 235 235 235 F F F 29 580-599 146 146 146 241 241 241 F F F Max gen: 147, Max flush: 242 Begin loop 74 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.201312948Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.201544494Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.201563913Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.201579212Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774} {"msg":"Crucible fa9f8b80-70db-490a-ad65-a67e199dcc4f has session id: 1270d4f4-10fa-4ec7-be17-65e2a5918f92","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.201869946Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"Upstairs opts: Upstairs UUID: fa9f8b80-70db-490a-ad65-a67e199dcc4f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.201918033Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"Crucible stats registered with UUID: fa9f8b80-70db-490a-ad65-a67e199dcc4f","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.201941242Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.233067349Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774} {"msg":"fa9f8b80-70db-490a-ad65-a67e199dcc4f active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.233098907Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.233117516Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"client":"0","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.233145435Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"client":"0","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.233162954Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"client":"1","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.233183743Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"client":"1","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.233219761Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"client":"2","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.233251579Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"client":"2","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.233276857Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.233304956Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.233337124Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"ds_connection connected from Ok(127.0.0.1:34922)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.233371682Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"ds_connection connected from Ok(127.0.0.1:40756)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.23340595Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"ds_connection connected from Ok(127.0.0.1:42630)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.233616008Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.234219514Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"client":"0","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.234246792Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"client":"0","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.23428994Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"client":"1","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.234307199Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"client":"1","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.234320738Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"client":"2","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.234334097Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"client":"2","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.234943713Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"client":"0","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.235090995Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"client":"1","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.235133742Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"client":"2","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.235472973Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"client":"0","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.235501261Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"client":"0","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.23551665Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.235708109Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"client":"1","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.235734698Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"client":"1","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.235749677Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.235762397Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"client":"2","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.235775696Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"client":"2","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"[0]R flush_numbers[0..12]: [236, 229, 241, 241, 241, 219, 242, 242, 241, 241, 241, 240]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.235789645Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"":"downstairs","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"[0]R generation[0..12]: [144, 138, 146, 146, 146, 131, 147, 147, 146, 146, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.235803684Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"":"downstairs","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.235817783Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"":"downstairs","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"[1]R flush_numbers[0..12]: [236, 229, 241, 241, 241, 219, 242, 242, 241, 241, 241, 240]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.235831773Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"":"downstairs","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"[1]R generation[0..12]: [144, 138, 146, 146, 146, 131, 147, 147, 146, 146, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.235845702Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"":"downstairs","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.235863031Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"":"downstairs","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"[2]R flush_numbers[0..12]: [236, 229, 241, 241, 241, 219, 242, 242, 241, 241, 241, 240]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.23587712Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"":"downstairs","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"[2]R generation[0..12]: [144, 138, 146, 146, 146, 131, 147, 147, 146, 146, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.235890979Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"":"downstairs","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.235904798Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"":"downstairs","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.235917728Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"":"downstairs","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.235930787Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"":"downstairs","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"Next flush: 243","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.235943416Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"":"downstairs","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.235956235Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"":"downstairs","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.235969705Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.235982024Z","hostname":Wait for a query_work_queue command to finish before sending IO "w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"client":"0","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.236013622Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"client":"1","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.236028431Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"client":"2","":"downstairs","session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.236041791Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.23605388Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"fa9f8b80-70db-490a-ad65-a67e199dcc4f is now active with session: 1270d4f4-10fa-4ec7-be17-65e2a5918f92","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.236066719Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.236080928Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774,"session_id":"1270d4f4-10fa-4ec7-be17-65e2a5918f92"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:32.236093148Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 442 len 8192 data: 13 14 02/30 Read block 181 len 24576 03/30 Write block 99 len 24576 data: 9 10 13 12 13 13 04/30 Read block 28 len 40960 05/30 Write block 567 len 32768 data: 11 14 14 14 17 18 18 16 06/30 Read block 66 len 32768 07/30 Read block 57 len 8192 08/30 Write block 527 len 24576 data: 13 13 11 11 13 14 09/30 Write block 258 len 4096 data: 8 10/30 Write block 504 len 20480 data: 17 15 13 13 12 11/30 Read block 146 len 16384 12/30 Read block 314 len 32768 13/30 Read block 77 len 40960 14/30 Write block 90 len 32768 data: 16 15 14 14 12 10 9 10 15/30 Read block 581 len 24576 16/30 Write block 579 len 4096 data: 11 17/30 Read block 247 len 12288 18/30 Write block 248 len 20480 data: 10 11 8 7 8 19/30 Read block 186 len 40960 20/30 Write block 377 len 4096 data: 10 21/30 Write block 149 len 8192 data: 7 7 22/30 Read block 161 len 12288 23/30 Read block 255 len 12288 24/30 Read block 82 len 4096 25/30 Read block 361 len 40960 26/30 Write block 499 len 20480 data: 14 12 14 15 16 27/30 Write block 426 len 12288 data: 10 10 9 28/30 Write block 299 len 32768 data: 12 11 7 13 17 16 19 14 29/30 Read block 378 len 16384 30/30 Write block 333 len 36864 data: 10 12 10 11 8 8 6 9 8 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Write 2 Done Done Done false 2 Acked 1001 Read 6 Done Done Done false 3 Acked 1002 Write 6 Done Done Done false 4 Acked 1003 Read 10 Done Done Done false 5 Acked 1004 Write 8 Done Done Done false 6 Acked 1005 Read 8 Done Done Done false 7 Acked 1006 Read 2 Done Done Done false 8 Acked 1007 Write 6 Done Done Done false 9 Acked 1008 Write 1 Done Done Done false 10 Acked 1009 Write 5 Done Done Done false 11 Acked 1010 Read 4 Done Done Done false 12 Acked 1011 Read 8 Done Done Done false 13 Acked 1012 Read 10 Done Done Done false 14 Acked 1013 Write 8 Done Done Done false 15 Acked 1014 Read 6 Done Done Done false 16 Acked 1015 Write 1 Done Done Done false 17 Acked 1016 Read 3 Done Done Done false 18 Acked 1017 Write 5 Done Done Done false 19 Acked 1018 Read 10 Done Done Done false 20 Acked 1019 Write 1 Done Done Done false 21 Acked 1020 Write 2 Done Done Done false 22 Acked 1021 Read 3 Done Done Done false 23 Acked 1022 Read 3 Done Done Done false 24 Acked 1023 Read 1 Done Done Done false 25 Acked 1024 Read 10 Done Done Done false 26 Acked 1025 Write 5 Done Done Done false 27 Acked 1026 Write 3 Done Done Done false 28 Acked 1027 Write 8 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:33:32.254811088Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1774} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 236 236 236 F F F 1 020-039 138 138 138 229 229 229 F F F 2 040-059 146 146 146 241 241 241 F F F 3 060-079 146 146 146 241 241 241 F F F 4 080-099 146 146 146 241 241 241 T T T 5 100-119 131 131 131 219 219 219 T T T 6 120-139 147 147 147 242 242 242 F F F 7 140-159 147 147 147 242 242 242 T T T 8 160-179 146 146 146 241 241 241 F F F 9 180-199 146 146 146 241 241 241 F F F 10 200-219 146 146 146 241 241 241 F F F 11 220-239 146 146 146 240 240 240 F F F 12 240-259 122 122 122 206 206 206 T T T 13 260-279 140 140 140 233 233 233 F F F 14 280-299 146 146 146 240 240 240 T T T 15 300-319 147 147 147 242 242 242 T T T 16 320-339 147 147 147 242 242 242 T T T 17 340-359 138 138 138 229 229 229 T T T 18 360-379 142 142 142 235 235 235 T T T 19 380-399 142 142 142 235 235 235 F F F 20 400-419 140 140 140 232 232 232 F F F 21 420-439 136 136 136 228 228 228 T T T 22 440-459 147 147 147 242 242 242 T T T 23 460-479 140 140 140 233 233 233 F F F 24 480-499 146 146 146 241 241 241 T T T 25 500-519 144 144 144 238 238 238 T T T 26 520-539 146 146 146 241 241 241 T T T 27 540-559 146 146 146 241 241 241 F F F 28 560-579 142 142 142 235 235 235 T T T 29 580-599 146 146 146 241 241 241 F F F Max gen: 147, Max flush: 242 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 236 236 236 F F F 1 020-039 138 138 138 229 229 229 F F F 2 040-059 146 146 146 241 241 241 F F F 3 060-079 146 146 146 241 241 241 F F F 4 080-099 148 148 148 243 243 243 F F F 5 100-119 148 148 148 243 243 243 F F F 6 120-139 147 147 147 242 242 242 F F F 7 140-159 148 148 148 243 243 243 F F F 8 160-179 146 146 146 241 241 241 F F F 9 180-199 146 146 146 241 241 241 F F F 10 200-219 146 146 146 241 241 241 F F F 11 220-239 146 146 146 240 240 240 F F F 12 240-259 148 148 148 243 243 243 F F F 13 260-279 140 140 140 233 233 233 F F F 14 280-299 148 148 148 243 243 243 F F F 15 300-319 148 148 148 243 243 243 F F F 16 320-339 148 148 148 243 243 243 F F F 17 340-359 148 148 148 243 243 243 F F F 18 360-379 148 148 148 243 243 243 F F F 19 380-399 142 142 142 235 235 235 F F F 20 400-419 140 140 140 232 232 232 F F F 21 420-439 148 148 148 243 243 243 F F F 22 440-459 148 148 148 243 243 243 F F F 23 460-479 140 140 140 233 233 233 F F F 24 480-499 148 148 148 243 243 243 F F F 25 500-519 148 148 148 243 243 243 F F F 26 520-539 148 148 148 243 243 243 F F F 27 540-559 146 146 146 241 241 241 F F F 28 560-579 148 148 148 243 243 243 F F F 29 580-599 146 146 146 241 241 241 F F F Max gen: 148, Max flush: 243 Begin loop 75 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.776353523Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.77657695Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.776595899Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.776611678Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782} {"msg":"Crucible db0d1224-c317-42b6-bcea-2b365f7de5ad has session id: 5ea59eeb-dedf-426b-81f5-f859537f0d89","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.776806667Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"Upstairs opts: Upstairs UUID: db0d1224-c317-42b6-bcea-2b365f7de5ad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.776850995Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"Crucible stats registered with UUID: db0d1224-c317-42b6-bcea-2b365f7de5ad","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.776867014Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.807739845Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782} {"msg":"db0d1224-c317-42b6-bcea-2b365f7de5ad active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.807764664Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.807779473Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"client":"0","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.807805282Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"client":"0","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.80783112Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"client":"1","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.807862238Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"client":"1","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.807887337Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"client":"2","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.807913416Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"client":"2","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.807937714Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.807963693Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.807988551Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"ds_connection connected from Ok(127.0.0.1:37405)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.80801737Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"ds_connection connected from Ok(127.0.0.1:56271)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.808045258Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"ds_connection connected from Ok(127.0.0.1:42456)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.808074336Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.808881731Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"client":"2","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.808909519Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"client":"2","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.808923978Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"client":"0","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.808938558Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"client":"0","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.808951797Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"client":"1","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.809010503Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"client":"1","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.809611159Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"client":"2","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.809651687Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"client":"0","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.809684125Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"client":"1","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.810223995Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"client":"1","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.810252003Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"client":"1","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.810270082Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.810346998Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"client":"0","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.810378246Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"client":"0","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.810392485Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.810405014Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"client":"2","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.810418404Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"client":"2","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"[0]R flush_numbers[0..12]: [236, 229, 241, 241, 243, 243, 242, 243, 241, 241, 241, 240]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.810432503Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"":"downstairs","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"[0]R generation[0..12]: [144, 138, 146, 146, 148, 148, 147, 148, 146, 146, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.810446442Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"":"downstairs","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.810460501Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"":"downstairs","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"[1]R flush_numbers[0..12]: [236, 229, 241, 241, 243, 243, 242, 243, 241, 241, 241, 240]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.810474231Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"":"downstairs","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"[1]R generation[0..12]: [144, 138, 146, 146, 148, 148, 147, 148, 146, 146, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.81049189Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"":"downstairs","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.810506479Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"":"downstairs","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"[2]R flush_numbers[0..12]: [236, 229, 241, 241, 243, 243, 242, 243, 241, 241, 241, 240]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.810520238Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"":"downstairs","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"[2]R generation[0..12]: [144, 138, 146, 146, 148, 148, 147, 148, 146, 146, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.810533737Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"":"downstairs","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.810547546Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"":"downstairs","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"Max found gen is 149","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.810560286Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"":"downstairs","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"Generation requested: 150 >= found:149","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.810573465Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"":"downstairs","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"Next flush: 244","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.810586374Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"":"downstairs","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.810599283Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.81066632Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.810682179Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"client":"0","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.810695548Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"client":"1","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.810708597Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"client":"2","":"downstairs","session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.810721726Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.810733856Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"db0d1224-c317-42b6-bcea-2b365f7de5ad is now active with session: 5ea59eeb-dedf-426b-81f5-f859537f0d89","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.810749205Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.810761364Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782,"session_id":"5ea59eeb-dedf-426b-81f5-f859537f0d89"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:36.810773554Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 369 len 20480 data: 6 8 9 9 9 03/30 Read block 420 len 24576 04/30 Write block 72 len 12288 data: 14 13 13 05/30 Read block 245 len 12288 06/30 Read block 519 len 12288 07/30 Write block 133 len 32768 data: 11 11 9 10 10 8 9 12 08/30 Read block 187 len 36864 09/30 Flush 10/30 Write block 450 len 32768 data: 10 9 8 8 10 11 12 12 11/30 Write block 461 len 16384 data: 9 6 8 8 12/30 Read block 523 len 36864 13/30 Write block 149 len 12288 data: 8 8 10 14/30 Read block 107 len 4096 15/30 Read block 221 len 24576 16/30 Write block 46 len 20480 data: 12 12 14 13 16 17/30 Write block 31 len 36864 data: 9 10 10 5 8 7 6 7 7 18/30 Flush 19/30 Write block 281 len 12288 data: 11 14 12 20/30 Write block 266 len 36864 data: 11 13 14 13 10 9 9 8 12 21/30 Read block 515 len 40960 22/30 Read block 403 len 16384 23/30 Read block 542 len 32768 24/30 Write block 184 len 24576 data: 15 14 10 12 15 13 25/30 Flush 26/30 Write block 367 len 40960 data: 8 7 7 9 10 10 10 11 11 9 27/30 Read block 71 len 28672 28/30 Write block 4 len 16384 data: 8 7 4 5 29/30 Read block 394 len 32768 30/30 Read block 407 len 36864 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 10 Done Done Done false 27 Acked 1026 Read 7 Done Done Done false 28 Acked 1027 Write 4 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 5 4 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:33:36.838219699Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1782} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 236 236 236 T T T 1 020-039 150 150 150 246 246 246 F F F 2 040-059 150 150 150 246 246 246 F F F 3 060-079 150 150 150 245 245 245 F F F 4 080-099 148 148 148 243 243 243 F F F 5 100-119 148 148 148 243 243 243 F F F 6 120-139 150 150 150 245 245 245 F F F 7 140-159 150 150 150 246 246 246 F F F 8 160-179 146 146 146 241 241 241 F F F 9 180-199 150 150 150 247 247 247 F F F 10 200-219 146 146 146 241 241 241 F F F 11 220-239 146 146 146 240 240 240 F F F 12 240-259 148 148 148 243 243 243 F F F 13 260-279 150 150 150 247 247 247 F F F 14 280-299 150 150 150 247 247 247 F F F 15 300-319 148 148 148 243 243 243 F F F 16 320-339 148 148 148 243 243 243 F F F 17 340-359 148 148 148 243 243 243 F F F 18 360-379 150 150 150 245 245 245 T T T 19 380-399 142 142 142 235 235 235 F F F 20 400-419 140 140 140 232 232 232 F F F 21 420-439 148 148 148 243 243 243 F F F 22 440-459 150 150 150 246 246 246 F F F 23 460-479 150 150 150 246 246 246 F F F 24 480-499 148 148 148 243 243 243 F F F 25 500-519 148 148 148 243 243 243 F F F 26 520-539 148 148 148 243 243 243 F F F 27 540-559 146 146 146 241 241 241 F F F 28 560-579 148 148 148 243 243 243 F F F 29 580-599 146 146 146 241 241 241 F F F Max gen: 150, Max flush: 247 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 248 248 248 F F F 1 020-039 150 150 150 246 246 246 F F F 2 040-059 150 150 150 246 246 246 F F F 3 060-079 150 150 150 245 245 245 F F F 4 080-099 148 148 148 243 243 243 F F F 5 100-119 148 148 148 243 243 243 F F F 6 120-139 150 150 150 245 245 245 F F F 7 140-159 150 150 150 246 246 246 F F F 8 160-179 146 146 146 241 241 241 F F F 9 180-199 150 150 150 247 247 247 F F F 10 200-219 146 146 146 241 241 241 F F F 11 220-239 146 146 146 240 240 240 F F F 12 240-259 148 148 148 243 243 243 F F F 13 260-279 150 150 150 247 247 247 F F F 14 280-299 150 150 150 247 247 247 F F F 15 300-319 148 148 148 243 243 243 F F F 16 320-339 148 148 148 243 243 243 F F F 17 340-359 148 148 148 243 243 243 F F F 18 360-379 151 151 151 248 248 248 F F F 19 380-399 142 142 142 235 235 235 F F F 20 400-419 140 140 140 232 232 232 F F F 21 420-439 148 148 148 243 243 243 F F F 22 440-459 150 150 150 246 246 246 F F F 23 460-479 150 150 150 246 246 246 F F F 24 480-499 148 148 148 243 243 243 F F F 25 500-519 148 148 148 243 243 243 F F F 26 520-539 148 148 148 243 243 243 F F F 27 540-559 146 146 146 241 241 241 F F F 28 560-579 148 148 148 243 243 243 F F F 29 580-599 146 146 146 241 241 241 F F F Max gen: 151, Max flush: 248 Begin loop 76 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.212261647Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.212586348Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.212626456Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.212658955Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790} {"msg":"Crucible 5038e5cf-7e73-4827-8632-8ae2e28eb7f4 has session id: d10ad4a5-2367-442b-8967-3a5190cee097","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.212685643Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"Upstairs opts: Upstairs UUID: 5038e5cf-7e73-4827-8632-8ae2e28eb7f4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.212750599Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"Crucible stats registered with UUID: 5038e5cf-7e73-4827-8632-8ae2e28eb7f4","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.212780868Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.243805451Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790} {"msg":"5038e5cf-7e73-4827-8632-8ae2e28eb7f4 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.243845598Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.243876316Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"client":"0","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.243921434Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"client":"0","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.243956192Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"client":"1","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.24398918Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"client":"1","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.244049797Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"client":"2","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.244087015Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"client":"2","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.244119413Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.244150621Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.24417759Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"ds_connection connected from Ok(127.0.0.1:44815)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.244319441Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"ds_connection connected from Ok(127.0.0.1:45128)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.24435114Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"ds_connection connected from Ok(127.0.0.1:52261)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.244375938Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.244946406Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"client":"1","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.244984424Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"client":"1","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.245014542Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"client":"0","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.24504179Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"client":"0","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.245069859Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"client":"2","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.245087178Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"client":"2","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.245598129Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"client":"1","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.245637667Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"client":"2","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.245668695Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"client":"0","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.246216584Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"client":"1","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.246248552Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"client":"1","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.246264611Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.246277871Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"client":"0","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.24629178Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"client":"0","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.246305219Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.246317518Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"client":"2","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.246330877Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"client":"2","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"[0]R flush_numbers[0..12]: [248, 246, 246, 245, 243, 243, 245, 246, 241, 247, 241, 240]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.246344727Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"":"downstairs","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"[0]R generation[0..12]: [151, 150, 150, 150, 148, 148, 150, 150, 146, 150, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.246358626Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"":"downstairs","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.246372665Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"":"downstairs","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"[1]R flush_numbers[0..12]: [248, 246, 246, 245, 243, 243, 245, 246, 241, 247, 241, 240]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.246386424Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"":"downstairs","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"[1]R generation[0..12]: [151, 150, 150, 150, 148, 148, 150, 150, 146, 150, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.246400193Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"":"downstairs","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.246414483Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"":"downstairs","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"[2]R flush_numbers[0..12]: [248, 246, 246, 245, 243, 243, 245, 246, 241, 247, 241, 240]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.246428432Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"":"downstairs","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"[2]R generation[0..12]: [151, 150, 150, 150, 148, 148, 150, 150, 146, 150, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.246442111Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"":"downstairs","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.246496328Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"":"downstairs","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.246513517Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"":"downstairs","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.246526826Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"":"downstairs","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"Next flush: 249","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.246539786Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"":"downstairs","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.246552595Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"":"downstairs","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.246565974Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.246578234Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"client":"0","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.246591883Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"client":"1","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.246605222Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"client":"2","":"downstairs","session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.246618301Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.246630611Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"5038e5cf-7e73-4827-8632-8ae2e28eb7f4 is now active with session: d10ad4a5-2367-442b-8967-3a5190cee097","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.24664335Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.246660219Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790,"session_id":"d10ad4a5-2367-442b-8967-3a5190cee097"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:41.246672648Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 1 len 24576 data: 8 7 9 9 8 5 02/30 Write block 513 len 16384 data: 9 10 8 8 03/30 Flush 04/30 Write block 174 len 40960 data: 11 11 11 8 8 11 12 10 13 15 05/30 Read block 394 len 12288 06/30 Read block 160 len 4096 07/30 Read block 384 len 20480 08/30 Read block 99 len 16384 09/30 Write block 431 len 28672 data: 10 11 12 11 14 14 12 10/30 Write block 87 len 12288 data: 15 13 14 11/30 Write block 191 len 16384 data: 11 13 10 9 12/30 Write block 381 len 12288 data: 7 9 10 13/30 Read block 54 len 28672 14/30 Read block 285 len 20480 15/30 Write block 546 len 20480 data: 19 19 19 16 16 16/30 Flush 17/30 Write block 165 len 20480 data: 8 7 11 10 12 18/30 Write block 542 len 8192 data: 15 16 19/30 Read block 178 len 36864 20/30 Write block 70 len 8192 data: 11 13 21/30 Flush 22/30 Write block 394 len 16384 data: 7 8 10 14 23/30 Read block 149 len 8192 24/30 Read block 482 len 12288 25/30 Write block 182 len 20480 data: 14 16 16 15 11 26/30 Read block 464 len 24576 27/30 Read block 397 len 40960 28/30 Read block 44 len 32768 29/30 Read block 314 len 12288 30/30 Write block 568 len 16384 data: 15 15 15 18 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 4 Done Done Done false 23 Acked 1022 Read 2 Done Done Done false 24 Acked 1023 Read 3 Done Done Done false 25 Acked 1024 Write 5 Done Done Done false 26 Acked 1025 Read 6 Done Done Done false 27 Acked 1026 Read 10 Done Done Done false 28 Acked 1027 Read 8 Done Done Done false 29 Acked 1028 Read 3 Done Done Done false 30 Acked 1029 Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:33:41.271379959Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1790} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 249 249 249 F F F 1 020-039 150 150 150 246 246 246 F F F 2 040-059 150 150 150 246 246 246 F F F 3 060-079 152 152 152 251 251 251 F F F 4 080-099 152 152 152 250 250 250 F F F 5 100-119 148 148 148 243 243 243 F F F 6 120-139 150 150 150 245 245 245 F F F 7 140-159 150 150 150 246 246 246 F F F 8 160-179 152 152 152 251 251 251 F F F 9 180-199 152 152 152 250 250 250 T T T 10 200-219 146 146 146 241 241 241 F F F 11 220-239 146 146 146 240 240 240 F F F 12 240-259 148 148 148 243 243 243 F F F 13 260-279 150 150 150 247 247 247 F F F 14 280-299 150 150 150 247 247 247 F F F 15 300-319 148 148 148 243 243 243 F F F 16 320-339 148 148 148 243 243 243 F F F 17 340-359 148 148 148 243 243 243 F F F 18 360-379 151 151 151 248 248 248 F F F 19 380-399 152 152 152 250 250 250 T T T 20 400-419 140 140 140 232 232 232 F F F 21 420-439 152 152 152 250 250 250 F F F 22 440-459 150 150 150 246 246 246 F F F 23 460-479 150 150 150 246 246 246 F F F 24 480-499 148 148 148 243 243 243 F F F 25 500-519 152 152 152 249 249 249 F F F 26 520-539 148 148 148 243 243 243 F F F 27 540-559 152 152 152 251 251 251 F F F 28 560-579 148 148 148 243 243 243 T T T 29 580-599 146 146 146 241 241 241 F F F Max gen: 152, Max flush: 251 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 249 249 249 F F F 1 020-039 150 150 150 246 246 246 F F F 2 040-059 150 150 150 246 246 246 F F F 3 060-079 152 152 152 251 251 251 F F F 4 080-099 152 152 152 250 250 250 F F F 5 100-119 148 148 148 243 243 243 F F F 6 120-139 150 150 150 245 245 245 F F F 7 140-159 150 150 150 246 246 246 F F F 8 160-179 152 152 152 251 251 251 F F F 9 180-199 153 153 153 252 252 252 F F F 10 200-219 146 146 146 241 241 241 F F F 11 220-239 146 146 146 240 240 240 F F F 12 240-259 148 148 148 243 243 243 F F F 13 260-279 150 150 150 247 247 247 F F F 14 280-299 150 150 150 247 247 247 F F F 15 300-319 148 148 148 243 243 243 F F F 16 320-339 148 148 148 243 243 243 F F F 17 340-359 148 148 148 243 243 243 F F F 18 360-379 151 151 151 248 248 248 F F F 19 380-399 153 153 153 252 252 252 F F F 20 400-419 140 140 140 232 232 232 F F F 21 420-439 152 152 152 250 250 250 F F F 22 440-459 150 150 150 246 246 246 F F F 23 460-479 150 150 150 246 246 246 F F F 24 480-499 148 148 148 243 243 243 F F F 25 500-519 152 152 152 249 249 249 F F F 26 520-539 148 148 148 243 243 243 F F F 27 540-559 152 152 152 251 251 251 F F F 28 560-579 153 153 153 252 252 252 F F F 29 580-599 146 146 146 241 241 241 F F F Max gen: 153, Max flush: 252 Begin loop 77 Downstairs to restart: 2 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.62437287Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.624681943Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.624708051Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.62473388Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798} {"msg":"Crucible 41e000bc-afb4-4205-ab46-70805435e16e has session id: 3fa73e90-934f-4000-aad9-0f8cc0e04614","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.624873662Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"Upstairs opts: Upstairs UUID: 41e000bc-afb4-4205-ab46-70805435e16e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.62491925Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"Crucible stats registered with UUID: 41e000bc-afb4-4205-ab46-70805435e16e","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.624946638Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.656202278Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798} {"msg":"41e000bc-afb4-4205-ab46-70805435e16e active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.656232686Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.656250345Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"client":"0","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.656277883Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"client":"0","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.656295393Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"client":"1","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.656312492Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"client":"1","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.656328901Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"client":"2","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.65634594Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"client":"2","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.656362479Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.656380018Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.656397357Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"ds_connection connected from Ok(127.0.0.1:58326)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.656605385Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"ds_connection connected from Ok(127.0.0.1:46439)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.656678231Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"ds_connection connected from Ok(127.0.0.1:58473)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.656802784Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.657467586Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"client":"1","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.657494935Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"client":"1","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.657509374Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"client":"2","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.657523283Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"client":"2","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.657536872Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"client":"0","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.657593569Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"client":"0","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.658215784Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"client":"1","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.658255791Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"client":"2","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.65828687Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"client":"0","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.658781932Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"client":"1","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.65880815Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"client":"1","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.658823039Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.658886756Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"client":"0","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.658917334Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"client":"0","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.658935173Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.65899613Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"client":"2","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.659024238Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"client":"2","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"[0]R flush_numbers[0..12]: [249, 246, 246, 251, 250, 243, 245, 246, 251, 252, 241, 240]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.659040757Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"":"downstairs","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"[0]R generation[0..12]: [152, 150, 150, 152, 152, 148, 150, 150, 152, 153, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.659055266Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"":"downstairs","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.659069525Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"":"downstairs","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"[1]R flush_numbers[0..12]: [249, 246, 246, 251, 250, 243, 245, 246, 251, 252, 241, 240]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.659083325Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"":"downstairs","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"[1]R generation[0..12]: [152, 150, 150, 152, 152, 148, 150, 150, 152, 153, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.659096924Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"":"downstairs","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.659110843Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"":"downstairs","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"[2]R flush_numbers[0..12]: [249, 246, 246, 251, 250, 243, 245, 246, 251, 252, 241, 240]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.659124862Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"":"downstairs","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"[2]R generation[0..12]: [152, 150, 150, 152, 152, 148, 150, 150, 152, 153, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.659138832Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"":"downstairs","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.659152911Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"":"downstairs","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.65916587Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"":"downstairs","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.659179309Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"":"downstairs","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"Next flush: 253","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.659192178Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"":"downstairs","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.659213607Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"":"downstairs","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-26T14:33:45.659234236Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.659256635Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"client":"0","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.659278813Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"client":"1","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.659292883Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"client":"2","":"downstairs","session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.659305972Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.659318321Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"41e000bc-afb4-4205-ab46-70805435e16e is now active with session: 3fa73e90-934f-4000-aad9-0f8cc0e04614","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.659331141Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.65934305Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798,"session_id":"3fa73e90-934f-4000-aad9-0f8cc0e04614"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:45.659355599Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798}Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 276 len 32768 data: 16 15 16 15 14 12 15 13 02/30 Write block 538 len 20480 data: 13 15 15 15 16 03/30 Write block 591 len 12288 data: 9 8 8 04/30 Read block 262 len 28672 05/30 Read block 48 len 24576 06/30 Write block 202 len 32768 data: 9 8 11 11 12 9 5 10 07/30 Read block 104 len 16384 08/30 Write block 570 len 8192 data: 16 19 09/30 Read block 108 len 20480 10/30 Write block 253 len 4096 data: 7 11/30 Write block 469 len 36864 data: 8 7 10 10 10 9 9 11 9 12/30 Write block 497 len 36864 data: 14 14 15 13 15 16 17 18 16 13/30 Write block 507 len 36864 data: 14 13 10 11 10 11 10 11 9 14/30 Read block 448 len 20480 15/30 Read block 348 len 36864 16/30 Flush 17/30 Flush 18/30 Write block 410 len 40960 data: 5 5 8 11 12 11 12 11 11 10 19/30 Flush 20/30 Flush 21/30 Write block 137 len 36864 data: 11 9 10 13 13 10 11 11 10 22/30 Read block 247 len 24576 23/30 Write block 107 len 32768 data: 11 13 12 11 15 16 18 16 24/30 Read block 19 len 24576 25/30 Flush 26/30 Flush 27/30 Write block 533 len 8192 data: 15 12 28/30 Read block 348 len 4096 29/30 Write block 332 len 20480 data: 10 11 13 11 12 30/30 Read block 110 len 32768 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 2 Done Done Done false 28 Acked 1027 Read 1 Done Done Done false 29 Acked 1028 Write 5 Done Done Done false 30 Acked 1029 Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 4 3 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:33:45.68864407Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1798} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 249 249 249 F F F 1 020-039 150 150 150 246 246 246 F F F 2 040-059 150 150 150 246 246 246 F F F 3 060-079 152 152 152 251 251 251 F F F 4 080-099 152 152 152 250 250 250 F F F 5 100-119 154 154 154 257 257 257 F F F 6 120-139 154 154 154 257 257 257 F F F 7 140-159 154 154 154 257 257 257 F F F 8 160-179 152 152 152 251 251 251 F F F 9 180-199 153 153 153 252 252 252 F F F 10 200-219 154 154 154 253 253 253 F F F 11 220-239 146 146 146 240 240 240 F F F 12 240-259 154 154 154 253 253 253 F F F 13 260-279 154 154 154 253 253 253 F F F 14 280-299 154 154 154 253 253 253 F F F 15 300-319 148 148 148 243 243 243 F F F 16 320-339 148 148 148 243 243 243 T T T 17 340-359 148 148 148 243 243 243 F F F 18 360-379 151 151 151 248 248 248 F F F 19 380-399 153 153 153 252 252 252 F F F 20 400-419 154 154 154 255 255 255 F F F 21 420-439 152 152 152 250 250 250 F F F 22 440-459 150 150 150 246 246 246 F F F 23 460-479 154 154 154 253 253 253 F F F 24 480-499 154 154 154 253 253 253 F F F 25 500-519 154 154 154 253 253 253 F F F 26 520-539 154 154 154 253 253 253 T T T 27 540-559 154 154 154 253 253 253 F F F 28 560-579 154 154 154 253 253 253 F F F 29 580-599 154 154 154 253 253 253 F F F Max gen: 154, Max flush: 257 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 249 249 249 F F F 1 020-039 150 150 150 246 246 246 F F F 2 040-059 150 150 150 246 246 246 F F F 3 060-079 152 152 152 251 251 251 F F F 4 080-099 152 152 152 250 250 250 F F F 5 100-119 154 154 154 257 257 257 F F F 6 120-139 154 154 154 257 257 257 F F F 7 140-159 154 154 154 257 257 257 F F F 8 160-179 152 152 152 251 251 251 F F F 9 180-199 153 153 153 252 252 252 F F F 10 200-219 154 154 154 253 253 253 F F F 11 220-239 146 146 146 240 240 240 F F F 12 240-259 154 154 154 253 253 253 F F F 13 260-279 154 154 154 253 253 253 F F F 14 280-299 154 154 154 253 253 253 F F F 15 300-319 148 148 148 243 243 243 F F F 16 320-339 155 155 155 258 258 258 F F F 17 340-359 148 148 148 243 243 243 F F F 18 360-379 151 151 151 248 248 248 F F F 19 380-399 153 153 153 252 252 252 F F F 20 400-419 154 154 154 255 255 255 F F F 21 420-439 152 152 152 250 250 250 F F F 22 440-459 150 150 150 246 246 246 F F F 23 460-479 154 154 154 253 253 253 F F F 24 480-499 154 154 154 253 253 253 F F F 25 500-519 154 154 154 253 253 253 F F F 26 520-539 155 155 155 258 258 258 F F F 27 540-559 154 154 154 253 253 253 F F F 28 560-579 154 154 154 253 253 253 F F F 29 580-599 154 154 154 253 253 253 F F F Max gen: 155, Max flush: 258 Begin loop 78 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.026707516Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.026954182Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.02697269Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.027036637Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806} {"msg":"Crucible 27ad71d4-c6cf-4862-9a2a-5b0378155988 has session id: 5c62b955-b78d-4e6f-a96b-b6aeb1aabb77","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.02716204Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"Upstairs opts: Upstairs UUID: 27ad71d4-c6cf-4862-9a2a-5b0378155988, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.027210327Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"Crucible stats registered with UUID: 27ad71d4-c6cf-4862-9a2a-5b0378155988","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.027233266Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.058812117Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806} {"msg":"27ad71d4-c6cf-4862-9a2a-5b0378155988 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.058850135Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.058879083Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"client":"0","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.05892946Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"client":"0","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.058964398Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"client":"1","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.058998407Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"client":"1","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.059032475Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"client":"2","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.059066343Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"client":"2","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.059099621Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.059210925Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.059244883Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"ds_connection connected from Ok(127.0.0.1:55257)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.059330798Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"ds_connection connected from Ok(127.0.0.1:63066)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.059356466Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"ds_connection connected from Ok(127.0.0.1:56428)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.059375525Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.059962272Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"client":"1","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.06000635Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"client":"1","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.060042197Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"client":"2","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.060074446Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"client":"2","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.060102714Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"client":"0","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.060141472Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"client":"0","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.060587447Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"client":"2","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.060713119Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"client":"1","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.060741858Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"client":"0","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.061175403Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"client":"2","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.061201912Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"client":"2","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.061216611Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.061325445Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"client":"1","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.061345574Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"client":"1","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.061359023Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.061371292Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"client":"0","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.061384932Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"client":"0","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"[0]R flush_numbers[0..12]: [249, 246, 246, 251, 250, 257, 257, 257, 251, 252, 253, 240]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.061400341Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"":"downstairs","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"[0]R generation[0..12]: [152, 150, 150, 152, 152, 154, 154, 154, 152, 153, 154, 146]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.06141433Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"":"downstairs","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.061428609Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"":"downstairs","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"[1]R flush_numbers[0..12]: [249, 246, 246, 251, 250, 257, 257, 257, 251, 252, 253, 240]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.061446578Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"":"downstairs","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"[1]R generation[0..12]: [152, 150, 150, 152, 152, 154, 154, 154, 152, 153, 154, 146]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.061461027Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"":"downstairs","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.061475116Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"":"downstairs","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"[2]R flush_numbers[0..12]: [249, 246, 246, 251, 250, 257, 257, 257, 251, 252, 253, 240]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.061488746Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"":"downstairs","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"[2]R generation[0..12]: [152, 150, 150, 152, 152, 154, 154, 154, 152, 153, 154, 146]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.061502415Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"":"downstairs","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.061516254Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"":"downstairs","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.061528993Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"":"downstairs","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.061542192Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"":"downstairs","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"Next flush: 259","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2024-09-26T14:33:50.061602369Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"":"downstairs","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.061623498Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"":"downstairs","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.061637527Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.061649457Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"client":"0","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.061663406Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"client":"1","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.061676515Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"client":"2","":"downstairs","session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.061692294Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.061704853Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"27ad71d4-c6cf-4862-9a2a-5b0378155988 is now active with session: 5c62b955-b78d-4e6f-a96b-b6aeb1aabb77","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.061717683Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.061729722Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806,"session_id":"5c62b955-b78d-4e6f-a96b-b6aeb1aabb77"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:50.061741921Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 490 len 32768 data: 11 11 13 13 14 14 14 15 02/30 Read block 169 len 12288 03/30 Flush 04/30 Write block 502 len 8192 data: 17 18 05/30 Read block 526 len 12288 06/30 Read block 329 len 16384 07/30 Write block 155 len 32768 data: 12 17 16 14 14 11 9 8 08/30 Flush 09/30 Write block 543 len 4096 data: 17 10/30 Read block 504 len 24576 11/30 Write block 195 len 8192 data: 10 11 12/30 Write block 92 len 4096 data: 15 13/30 Read block 80 len 36864 14/30 Read block 268 len 28672 15/30 Write block 379 len 12288 data: 9 10 8 16/30 Write block 135 len 16384 data: 10 11 12 10 17/30 Read block 350 len 24576 18/30 Read block 377 len 36864 19/30 Write block 404 len 32768 data: 8 7 8 4 5 4 6 6 20/30 Read block 33 len 4096 21/30 Read block 175 len 20480 22/30 Write block 499 len 32768 data: 16 14 16 18 19 19 17 14 23/30 Read block 355 len 20480 24/30 Read block 9 len 20480 25/30 Read block 100 len 24576 26/30 Write block 231 len 24576 data: 4 4 6 5 4 4 27/30 Read block 233 len 32768 28/30 Read block 372 len 32768 29/30 Read block 273 len 20480 30/30 Write block 19 len 20480 data: 11 8 7 10 12 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 9 Acked 1008 Write 1 Done Done Done false 10 Acked 1009 Read 6 Done Done Done false 11 Acked 1010 Write 2 Done Done Done false 12 Acked 1011 Write 1 Done Done Done false 13 Acked 1012 Read 9 Done Done Done false 14 Acked 1013 Read 7 Done Done Done false 15 Acked 1014 Write 3 Done Done Done false 16 Acked 1015 Write 4 Done Done Done false 17 Acked 1016 Read 6 Done Done Done false 18 Acked 1017 Read 9 Done Done Done false 19 Acked 1018 Write 8 Done Done Done false 20 Acked 1019 Read 1 Done Done Done false 21 Acked 1020 Read 5 Done Done Done false 22 Acked 1021 Write 8 Done Done Done false 23 Acked 1022 Read 5 Done Done Done false 24 Acked 1023 Read 5 Done Done Done false 25 Acked 1024 Read 6 Done Done Done false 26 Acked 1025 Write 6 Done Done Done false 27 Acked 1026 Read 8 Done Done Done false 28 Acked 1027 Read 8 Done Done Done false 29 Acked 1028 Read 5 Done Done Done false 30 Acked 1029 Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 21 21 21 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1007 1007 1007 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:33:50.084795546Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1806} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 249 249 249 T T T 1 020-039 150 150 150 246 246 246 T T T 2 040-059 150 150 150 246 246 246 F F F 3 060-079 152 152 152 251 251 251 F F F 4 080-099 152 152 152 250 250 250 T T T 5 100-119 154 154 154 257 257 257 F F F 6 120-139 154 154 154 257 257 257 T T T 7 140-159 156 156 156 260 260 260 F F F 8 160-179 156 156 156 260 260 260 F F F 9 180-199 153 153 153 252 252 252 T T T 10 200-219 154 154 154 253 253 253 F F F 11 220-239 146 146 146 240 240 240 T T T 12 240-259 154 154 154 253 253 253 F F F 13 260-279 154 154 154 253 253 253 F F F 14 280-299 154 154 154 253 253 253 F F F 15 300-319 148 148 148 243 243 243 F F F 16 320-339 155 155 155 258 258 258 F F F 17 340-359 148 148 148 243 243 243 F F F 18 360-379 151 151 151 248 248 248 T T T 19 380-399 153 153 153 252 252 252 T T T 20 400-419 154 154 154 255 255 255 T T T 21 420-439 152 152 152 250 250 250 F F F 22 440-459 150 150 150 246 246 246 F F F 23 460-479 154 154 154 253 253 253 F F F 24 480-499 156 156 156 259 259 259 T T T 25 500-519 156 156 156 260 260 260 T T T 26 520-539 155 155 155 258 258 258 F F F 27 540-559 154 154 154 253 253 253 T T T 28 560-579 154 154 154 253 253 253 F F F 29 580-599 154 154 154 253 253 253 F F F Max gen: 156, Max flush: 260 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 261 261 261 F F F 1 020-039 157 157 157 261 261 261 F F F 2 040-059 150 150 150 246 246 246 F F F 3 060-079 152 152 152 251 251 251 F F F 4 080-099 157 157 157 261 261 261 F F F 5 100-119 154 154 154 257 257 257 F F F 6 120-139 157 157 157 261 261 261 F F F 7 140-159 156 156 156 260 260 260 F F F 8 160-179 156 156 156 260 260 260 F F F 9 180-199 157 157 157 261 261 261 F F F 10 200-219 154 154 154 253 253 253 F F F 11 220-239 157 157 157 261 261 261 F F F 12 240-259 154 154 154 253 253 253 F F F 13 260-279 154 154 154 253 253 253 F F F 14 280-299 154 154 154 253 253 253 F F F 15 300-319 148 148 148 243 243 243 F F F 16 320-339 155 155 155 258 258 258 F F F 17 340-359 148 148 148 243 243 243 F F F 18 360-379 157 157 157 261 261 261 F F F 19 380-399 157 157 157 261 261 261 F F F 20 400-419 157 157 157 261 261 261 F F F 21 420-439 152 152 152 250 250 250 F F F 22 440-459 150 150 150 246 246 246 F F F 23 460-479 154 154 154 253 253 253 F F F 24 480-499 157 157 157 261 261 261 F F F 25 500-519 157 157 157 261 261 261 F F F 26 520-539 155 155 155 258 258 258 F F F 27 540-559 157 157 157 261 261 261 F F F 28 560-579 154 154 154 253 253 253 F F F 29 580-599 154 154 154 253 253 253 F F F Max gen: 157, Max flush: 261 Begin loop 79 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.551844588Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.552253505Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.552288513Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.552321591Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814} {"msg":"Crucible 297408ff-bfd6-4a7c-ad37-7981d563ec5f has session id: db2f15f8-ad63-41e7-b133-c54f7c6b2f2a","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.552349959Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"Upstairs opts: Upstairs UUID: 297408ff-bfd6-4a7c-ad37-7981d563ec5f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.552423905Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"Crucible stats registered with UUID: 297408ff-bfd6-4a7c-ad37-7981d563ec5f","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.552454173Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.583807258Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814} {"msg":"297408ff-bfd6-4a7c-ad37-7981d563ec5f active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.583835126Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.583853195Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"client":"0","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.583880133Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"client":"0","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.583897403Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"client":"1","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.583919161Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"client":"1","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.58393682Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"client":"2","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.583956519Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"client":"2","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.584030185Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.584060963Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.584089292Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:37147)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.5841166Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:37309)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.584142259Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:64776)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.584307509Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.584894966Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"client":"0","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.584932314Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"client":"0","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.584979211Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"client":"2","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.585057367Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"client":"2","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.585093405Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"client":"1","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.585125373Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"client":"1","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.58553387Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"client":"2","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.585565948Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"client":"1","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.58569985Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"client":"0","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.586139306Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"client":"2","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.586164984Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"client":"2","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.586178903Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.586257219Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"client":"1","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.586276158Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"client":"1","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.586289747Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.586301916Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"client":"0","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.586315156Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"client":"0","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"[0]R flush_numbers[0..12]: [261, 261, 246, 251, 261, 257, 261, 260, 260, 261, 253, 261]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.586333055Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"":"downstairs","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"[0]R generation[0..12]: [157, 157, 150, 152, 157, 154, 157, 156, 156, 157, 154, 157]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.586347024Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"":"downstairs","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.586361133Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"":"downstairs","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"[1]R flush_numbers[0..12]: [261, 261, 246, 251, 261, 257, 261, 260, 260, 261, 253, 261]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.586374882Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"":"downstairs","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"[1]R generation[0..12]: [157, 157, 150, 152, 157, 154, 157, 156, 156, 157, 154, 157]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.586392751Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"":"downstairs","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.586407161Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"":"downstairs","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"[2]R flush_numbers[0..12]: [261, 261, 246, 251, 261, 257, 261, 260, 260, 261, 253, 261]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.58642096Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"":"downstairs","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"[2]R generation[0..12]: [157, 157, 150, 152, 157, 154, 157, 156, 156, 157, 154, 157]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.586435419Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"":"downstairs","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.586449298Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"":"downstairs","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.586462257Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"":"downstairs","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.586475417Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"":"downstairs","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"Next flush: 262","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.586488106Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"":"downstairs","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.586500875Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"":"downstairs","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.586513994Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-26T14:33:54.586525834Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"client":"0","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.586539013Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"client":"1","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.586551952Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"client":"2","":"downstairs","session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.586565041Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.586577301Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"297408ff-bfd6-4a7c-ad37-7981d563ec5f is now active with session: db2f15f8-ad63-41e7-b133-c54f7c6b2f2a","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.5865918Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.586603969Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814,"session_id":"db2f15f8-ad63-41e7-b133-c54f7c6b2f2a"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:54.586616129Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 432 len 12288 data: 12 13 12 02/30 Write block 546 len 4096 data: 20 03/30 Flush 04/30 Write block 230 len 16384 data: 8 5 5 7 05/30 Write block 155 len 40960 data: 13 18 17 15 15 12 10 9 8 8 06/30 Write block 190 len 12288 data: 12 12 14 07/30 Read block 383 len 12288 08/30 Write block 67 len 36864 data: 12 12 10 12 14 15 14 14 10 09/30 Write block 329 len 12288 data: 10 11 12 10/30 Write block 397 len 32768 data: 15 15 15 13 15 14 12 9 11/30 Write block 171 len 12288 data: 15 14 13 12/30 Read block 531 len 28672 13/30 Read block 41 len 20480 14/30 Read block 52 len 20480 15/30 Write block 296 len 4096 data: 10 16/30 Read block 465 len 40960 17/30 Write block 54 len 40960 data: 8 9 10 12 11 11 11 10 8 10 18/30 Flush 19/30 Write block 20 len 36864 data: 9 8 11 13 12 10 12 12 15 20/30 Write block 167 len 24576 data: 12 11 13 14 16 15 21/30 Write block 247 len 40960 data: 11 11 12 9 8 9 8 10 10 9 22/30 Read block 315 len 40960 23/30 Flush 24/30 Write block 276 len 36864 data: 17 16 17 16 15 13 16 14 13 25/30 Read block 306 len 24576 26/30 Read block 2 len 32768 27/30 Write block 422 len 16384 data: 12 11 10 11 28/30 Write block 183 len 24576 data: 17 17 16 12 13 16 29/30 Write block 584 len 8192 data: 11 13 30/30 Write block 165 len 36864 data: 9 8 13 12 14 15 17 16 14 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 9 Done Done Done false 25 Acked 1024 Read 6 Done Done Done false 26 Acked 1025 Read 8 Done Done Done false 27 Acked 1026 Write 4 Done Done Done false 28 Acked 1027 Write 6 Done Done Done false 29 Acked 1028 Write 2 Done Done Sent false 30 Acked 1029 Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 6 6 5 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:33:54.610856916Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1814} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 261 261 261 F F F 1 020-039 158 158 158 264 264 264 F F F 2 040-059 158 158 158 263 263 263 F F F 3 060-079 158 158 158 263 263 263 F F F 4 080-099 157 157 157 261 261 261 F F F 5 100-119 154 154 154 257 257 257 F F F 6 120-139 157 157 157 261 261 261 F F F 7 140-159 158 158 158 263 263 263 F F F 8 160-179 158 158 158 264 264 264 T T T 9 180-199 158 158 158 263 263 263 T T T 10 200-219 154 154 154 253 253 253 F F F 11 220-239 158 158 158 263 263 263 F F F 12 240-259 158 158 158 264 264 264 F F F 13 260-279 154 154 154 253 253 253 T T T 14 280-299 158 158 158 263 263 263 T T T 15 300-319 148 148 148 243 243 243 F F F 16 320-339 158 158 158 263 263 263 F F F 17 340-359 148 148 148 243 243 243 F F F 18 360-379 157 157 157 261 261 261 F F F 19 380-399 158 158 158 263 263 263 F F F 20 400-419 158 158 158 263 263 263 F F F 21 420-439 158 158 158 262 262 262 T T T 22 440-459 150 150 150 246 246 246 F F F 23 460-479 154 154 154 253 253 253 F F F 24 480-499 157 157 157 261 261 261 F F F 25 500-519 157 157 157 261 261 261 F F F 26 520-539 155 155 155 258 258 258 F F F 27 540-559 158 158 158 262 262 262 F F F 28 560-579 154 154 154 253 253 253 F F F 29 580-599 154 154 154 253 253 253 T T T Max gen: 158, Max flush: 264 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 261 261 261 F F F 1 020-039 158 158 158 264 264 264 F F F 2 040-059 158 158 158 263 263 263 F F F 3 060-079 158 158 158 263 263 263 F F F 4 080-099 157 157 157 261 261 261 F F F 5 100-119 154 154 154 257 257 257 F F F 6 120-139 157 157 157 261 261 261 F F F 7 140-159 158 158 158 263 263 263 F F F 8 160-179 159 159 159 265 265 265 F F F 9 180-199 159 159 159 265 265 265 F F F 10 200-219 154 154 154 253 253 253 F F F 11 220-239 158 158 158 263 263 263 F F F 12 240-259 158 158 158 264 264 264 F F F 13 260-279 159 159 159 265 265 265 F F F 14 280-299 159 159 159 265 265 265 F F F 15 300-319 148 148 148 243 243 243 F F F 16 320-339 158 158 158 263 263 263 F F F 17 340-359 148 148 148 243 243 243 F F F 18 360-379 157 157 157 261 261 261 F F F 19 380-399 158 158 158 263 263 263 F F F 20 400-419 158 158 158 263 263 263 F F F 21 420-439 159 159 159 265 265 265 F F F 22 440-459 150 150 150 246 246 246 F F F 23 460-479 154 154 154 253 253 253 F F F 24 480-499 157 157 157 261 261 261 F F F 25 500-519 157 157 157 261 261 261 F F F 26 520-539 155 155 155 258 258 258 F F F 27 540-559 158 158 158 262 262 262 F F F 28 560-579 154 154 154 253 253 253 F F F 29 580-599 159 159 159 265 265 265 F F F Max gen: 159, Max flush: 265 Begin loop 80 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.010230209Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.010521293Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.010559201Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.010589749Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822} {"msg":"Crucible 5108cb08-3f24-4fdb-9eb2-d081439eb880 has session id: 90baa150-1d01-4522-81c7-76585e5070b3","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.010664535Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"Upstairs opts: Upstairs UUID: 5108cb08-3f24-4fdb-9eb2-d081439eb880, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.010716212Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"Crucible stats registered with UUID: 5108cb08-3f24-4fdb-9eb2-d081439eb880","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.01074213Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.042008189Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822} {"msg":"5108cb08-3f24-4fdb-9eb2-d081439eb880 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.042042007Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.042060097Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"client":"0","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.042087585Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"client":"0","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.042112283Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"client":"1","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.042143832Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"client":"1","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.04217733Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"client":"2","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.042210588Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"client":"2","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.042241176Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.042273714Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.042306083Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"ds_connection connected from Ok(127.0.0.1:55358)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.042336531Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"ds_connection connected from Ok(127.0.0.1:38833)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.042371629Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"ds_connection connected from Ok(127.0.0.1:38893)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.042503581Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.04305577Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"client":"1","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.043089528Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"client":"1","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.043111027Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"client":"2","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.043142305Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"client":"2","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.043304266Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"client":"0","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.043343314Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"client":"0","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.043842736Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"client":"2","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.043871304Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"client":"0","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.043886593Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"client":"1","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.04446354Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"client":"1","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.044491359Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"client":"1","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.044507068Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.044520077Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"client":"2","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.044535126Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"client":"2","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.044548816Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.044561165Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"client":"0","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.044574344Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"client":"0","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"[0]R flush_numbers[0..12]: [261, 264, 263, 263, 261, 257, 261, 263, 265, 265, 253, 263]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.044597793Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"":"downstairs","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"[0]R generation[0..12]: [157, 158, 158, 158, 157, 154, 157, 158, 159, 159, 154, 158]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.044612722Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"":"downstairs","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.044626871Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"":"downstairs","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"[1]R flush_numbers[0..12]: [261, 264, 263, 263, 261, 257, 261, 263, 265, 265, 253, 263]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.04464058Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"":"downstairs","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"[1]R generation[0..12]: [157, 158, 158, 158, 157, 154, 157, 158, 159, 159, 154, 158]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.04465441Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"":"downstairs","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.044668379Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"":"downstairs","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"[2]R flush_numbers[0..12]: [261, 264, 263, 263, 261, 257, 261, 263, 265, 265, 253, 263]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.044682088Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"":"downstairs","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"[2]R generation[0..12]: [157, 158, 158, 158, 157, 154, 157, 158, 159, 159, 154, 158]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.044699267Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"":"downstairs","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.044713536Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"":"downstairs","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.044726366Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"":"downstairs","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"timeWait for a query_work_queue command to finish before sending IO ":"2024-09-26T14:33:59.044739475Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"":"downstairs","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"Next flush: 266","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.044756014Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"":"downstairs","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.044769053Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"":"downstairs","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.044782362Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.044794432Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"client":"0","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.044807761Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"client":"1","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.04482113Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"client":"2","":"downstairs","session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.044834339Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.044846669Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"5108cb08-3f24-4fdb-9eb2-d081439eb880 is now active with session: 90baa150-1d01-4522-81c7-76585e5070b3","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.044859528Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.044871597Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822,"session_id":"90baa150-1d01-4522-81c7-76585e5070b3"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:33:59.044883877Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 589 len 32768 data: 13 13 10 9 9 9 8 7 03/30 Write block 516 len 36864 data: 9 8 10 11 11 9 7 13 13 04/30 Write block 427 len 4096 data: 11 05/30 Write block 427 len 36864 data: 12 10 9 12 11 13 14 13 15 06/30 Read block 208 len 40960 07/30 Flush 08/30 Write block 222 len 20480 data: 11 11 14 13 12 09/30 Write block 550 len 28672 data: 17 14 11 10 6 5 7 10/30 Read block 530 len 24576 11/30 Read block 465 len 28672 12/30 Write block 437 len 36864 data: 13 12 10 12 12 14 15 16 16 13/30 Write block 594 len 12288 data: 10 9 8 14/30 Read block 282 len 40960 15/30 Read block 208 len 12288 16/30 Read block 377 len 40960 17/30 Read block 297 len 12288 18/30 Write block 99 len 16384 data: 10 11 14 13 19/30 Read block 226 len 28672 20/30 Write block 523 len 20480 data: 14 14 13 15 14 21/30 Write block 363 len 28672 data: 15 14 10 8 9 8 8 22/30 Write block 403 len 12288 data: 13 10 8 23/30 Read block 252 len 36864 24/30 Write block 506 len 28672 data: 15 15 14 11 12 11 12 25/30 Read block 37 len 20480 26/30 Read block 223 len 40960 27/30 Write block 327 len 12288 data: 13 14 11 28/30 Read block 209 len 32768 29/30 Read block 178 len 8192 30/30 Write block 21 len 12288 data: 9 12 14 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 8 Acked 1007 Write 5 Done Done Done false 9 Acked 1008 Write 7 Done Done Done false 10 Acked 1009 Read 6 Done Done Done false 11 Acked 1010 Read 7 Done Done Done false 12 Acked 1011 Write 9 Done Done Done false 13 Acked 1012 Write 3 Done Done Done false 14 Acked 1013 Read 10 Done Done Done false 15 Acked 1014 Read 3 Done Done Done false 16 Acked 1015 Read 10 Done Done Done false 17 Acked 1016 Read 3 Done Done Done false 18 Acked 1017 Write 4 Done Done Done false 19 Acked 1018 Read 7 Done Done Done false 20 Acked 1019 Write 5 Done Done Done false 21 Acked 1020 Write 7 Done Done Done false 22 Acked 1021 Write 3 Done Done Done false 23 Acked 1022 Read 9 Done Done Done false 24 Acked 1023 Write 7 Done Done Done false 25 Acked 1024 Read 5 Done Done Done false 26 Acked 1025 Read 10 Done Done Done false 27 Acked 1026 Write 3 Done Done Done false 28 Acked 1027 Read 8 Done Done Sent false 29 Acked 1028 Read 2 Done Done Sent false 30 Acked 1029 Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 3 5 Done 22 22 20 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1006 1006 1006 Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:33:59.067135076Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1822} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 261 261 261 F F F 1 020-039 158 158 158 264 264 264 T T T 2 040-059 158 158 158 263 263 263 F F F 3 060-079 158 158 158 263 263 263 F F F 4 080-099 157 157 157 261 261 261 T T T 5 100-119 154 154 154 257 257 257 T T T 6 120-139 157 157 157 261 261 261 F F F 7 140-159 158 158 158 263 263 263 F F F 8 160-179 159 159 159 265 265 265 F F F 9 180-199 159 159 159 265 265 265 F F F 10 200-219 154 154 154 253 253 253 F F F 11 220-239 158 158 158 263 263 263 T T T 12 240-259 158 158 158 264 264 264 F F F 13 260-279 159 159 159 265 265 265 F F F 14 280-299 159 159 159 265 265 265 F F F 15 300-319 148 148 148 243 243 243 F F F 16 320-339 158 158 158 263 263 263 T T T 17 340-359 148 148 148 243 243 243 F F F 18 360-379 157 157 157 261 261 261 T T T 19 380-399 158 158 158 263 263 263 F F F 20 400-419 158 158 158 263 263 263 T T T 21 420-439 160 160 160 267 267 267 T T T 22 440-459 150 150 150 246 246 246 T T T 23 460-479 154 154 154 253 253 253 F F F 24 480-499 157 157 157 261 261 261 F F F 25 500-519 160 160 160 267 267 267 T T T 26 520-539 160 160 160 267 267 267 T T T 27 540-559 158 158 158 262 262 262 T T T 28 560-579 154 154 154 253 253 253 F F F 29 580-599 160 160 160 267 267 267 T T T Max gen: 160, Max flush: 267 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 261 261 261 F F F 1 020-039 161 161 161 268 268 268 F F F 2 040-059 158 158 158 263 263 263 F F F 3 060-079 158 158 158 263 263 263 F F F 4 080-099 161 161 161 268 268 268 F F F 5 100-119 161 161 161 268 268 268 F F F 6 120-139 157 157 157 261 261 261 F F F 7 140-159 158 158 158 263 263 263 F F F 8 160-179 159 159 159 265 265 265 F F F 9 180-199 159 159 159 265 265 265 F F F 10 200-219 154 154 154 253 253 253 F F F 11 220-239 161 161 161 268 268 268 F F F 12 240-259 158 158 158 264 264 264 F F F 13 260-279 159 159 159 265 265 265 F F F 14 280-299 159 159 159 265 265 265 F F F 15 300-319 148 148 148 243 243 243 F F F 16 320-339 161 161 161 268 268 268 F F F 17 340-359 148 148 148 243 243 243 F F F 18 360-379 161 161 161 268 268 268 F F F 19 380-399 158 158 158 263 263 263 F F F 20 400-419 161 161 161 268 268 268 F F F 21 420-439 161 161 161 268 268 268 F F F 22 440-459 161 161 161 268 268 268 F F F 23 460-479 154 154 154 253 253 253 F F F 24 480-499 157 157 157 261 261 261 F F F 25 500-519 161 161 161 268 268 268 F F F 26 520-539 161 161 161 268 268 268 F F F 27 540-559 161 161 161 268 268 268 F F F 28 560-579 154 154 154 253 253 253 F F F 29 580-599 161 161 161 268 268 268 F F F Max gen: 161, Max flush: 268 Begin loop 81 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.558909417Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.55920886Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.559252017Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.559281026Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830} {"msg":"Crucible 847ccf15-51a7-4437-abdb-52897ca3e697 has session id: bff53ca1-2d17-4590-8273-ddb0f0053a7c","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.559427798Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"Upstairs opts: Upstairs UUID: 847ccf15-51a7-4437-abdb-52897ca3e697, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.559475295Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"Crucible stats registered with UUID: 847ccf15-51a7-4437-abdb-52897ca3e697","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.559499334Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.590407643Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830} {"msg":"847ccf15-51a7-4437-abdb-52897ca3e697 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.590437741Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.5904554Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"client":"0","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.590483199Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"client":"0","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.590500468Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"client":"1","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.590517487Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"client":"1","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.590544575Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"client":"2","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.590578173Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"client":"2","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.590610272Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.59064525Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.590676508Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"ds_connection connected from Ok(127.0.0.1:59077)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.590707226Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"ds_connection connected from Ok(127.0.0.1:44700)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.590738214Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"ds_connection connected from Ok(127.0.0.1:50979)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.590881206Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.591480222Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"client":"0","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.59152032Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"client":"0","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.591550268Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"client":"2","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.591579787Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"client":"2","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.591607685Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"client":"1","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.591634403Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"client":"1","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.592161144Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"client":"0","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.592202282Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"client":"2","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.592250919Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"client":"1","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.592808267Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"client":"0","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.592838255Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"client":"0","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.592853154Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.592866964Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"client":"1","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.592880593Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"client":"1","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.592893702Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.592906091Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"client":"2","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.592919461Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"client":"2","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"[0]R flush_numbers[0..12]: [261, 268, 263, 263, 268, 268, 261, 263, 265, 265, 253, 268]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.59293344Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"":"downstairs","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"[0]R generation[0..12]: [157, 161, 158, 158, 161, 161, 157, 158, 159, 159, 154, 161]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.592947619Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"":"downstairs","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.592961758Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"":"downstairs","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"[1]R flush_numbers[0..12]: [261, 268, 263, 263, 268, 268, 261, 263, 265, 265, 253, 268]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.592975518Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"":"downstairs","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"[1]R generation[0..12]: [157, 161, 158, 158, 161, 161, 157, 158, 159, 159, 154, 161]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.592989257Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"":"downstairs","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.593020115Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"":"downstairs","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"[2]R flush_numbers[0..12]: [261, 268, 263, 263, 268, 268, 261, 263, 265, 265, 253, 268]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.593036494Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"":"downstairs","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"[2]R generation[0..12]: [157, 161, 158, 158, 161, 161, 157, 158, 159, 159, 154, 161]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.593050363Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"":"downstairs","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.593066693Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"":"downstairs","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-26T14:34:03.593079782Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"":"downstairs","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.593094371Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"":"downstairs","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"Next flush: 269","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.59310757Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"":"downstairs","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.593120579Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"":"downstairs","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.593134079Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.593145988Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"client":"0","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.593159457Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"client":"1","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.593172767Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"client":"2","":"downstairs","session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.593186256Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.593198565Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"847ccf15-51a7-4437-abdb-52897ca3e697 is now active with session: bff53ca1-2d17-4590-8273-ddb0f0053a7c","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.593211534Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.593223633Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830,"session_id":"bff53ca1-2d17-4590-8273-ddb0f0053a7c"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:03.593235823Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 485 len 32768 data: 10 10 10 9 10 12 12 14 02/30 Flush 03/30 Flush 04/30 Write block 490 len 28672 data: 13 13 15 14 15 15 15 05/30 Read block 309 len 20480 06/30 Read block 14 len 12288 07/30 Write block 268 len 8192 data: 15 14 08/30 Read block 375 len 40960 09/30 Read block 449 len 36864 10/30 Read block 514 len 12288 11/30 Read block 78 len 12288 12/30 Write block 573 len 4096 data: 19 13/30 Write block 352 len 12288 data: 15 17 15 14/30 Flush 15/30 Write block 437 len 8192 data: 14 13 16/30 Write block 19 len 40960 data: 12 10 10 13 15 13 11 13 13 16 17/30 Read block 474 len 12288 18/30 Flush 19/30 Write block 343 len 32768 data: 9 8 9 9 10 10 9 11 20/30 Write block 430 len 28672 data: 13 12 14 15 14 16 15 21/30 Read block 571 len 16384 22/30 Read block 37 len 20480 23/30 Read block 1 len 4096 24/30 Read block 183 len 8192 25/30 Write block 208 len 4096 data: 6 26/30 Write block 389 len 32768 data: 9 9 10 10 8 8 9 11 27/30 Write block 580 len 28672 data: 7 7 9 11 12 14 14 28/30 Write block 527 len 28672 data: 15 14 12 12 14 15 16 29/30 Read block 443 len 4096 30/30 Write block 552 len 28672 data: 12 11 7 6 8 6 7 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 19 Acked 1018 Write 8 Done Done Done false 20 Acked 1019 Write 7 Done Done Done false 21 Acked 1020 Read 4 Done Done Done false 22 Acked 1021 Read 5 Done Done Done false 23 Acked 1022 Read 1 Done Done Done false 24 Acked 1023 Read 2 Done Done Done false 25 Acked 1024 Write 1 Done Done Done false 26 Acked 1025 Write 8 Done Done Done false 27 Acked 1026 Write 7 Done Done Done false 28 Acked 1027 Write 7 Done Done Done false 29 Acked 1028 Read 1 Done Done Done false 30 Acked 1029 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:34:03.619384502Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1830} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 272 272 272 F F F 1 020-039 162 162 162 272 272 272 F F F 2 040-059 158 158 158 263 263 263 F F F 3 060-079 158 158 158 263 263 263 F F F 4 080-099 161 161 161 268 268 268 F F F 5 100-119 161 161 161 268 268 268 F F F 6 120-139 157 157 157 261 261 261 F F F 7 140-159 158 158 158 263 263 263 F F F 8 160-179 159 159 159 265 265 265 F F F 9 180-199 159 159 159 265 265 265 F F F 10 200-219 154 154 154 253 253 253 T T T 11 220-239 161 161 161 268 268 268 F F F 12 240-259 158 158 158 264 264 264 F F F 13 260-279 162 162 162 271 271 271 F F F 14 280-299 159 159 159 265 265 265 F F F 15 300-319 148 148 148 243 243 243 F F F 16 320-339 161 161 161 268 268 268 F F F 17 340-359 162 162 162 271 271 271 T T T 18 360-379 161 161 161 268 268 268 F F F 19 380-399 158 158 158 263 263 263 T T T 20 400-419 161 161 161 268 268 268 F F F 21 420-439 162 162 162 272 272 272 T T T 22 440-459 161 161 161 268 268 268 F F F 23 460-479 154 154 154 253 253 253 F F F 24 480-499 162 162 162 271 271 271 F F F 25 500-519 161 161 161 268 268 268 F F F 26 520-539 161 161 161 268 268 268 T T T 27 540-559 161 161 161 268 268 268 T T T 28 560-579 162 162 162 271 271 271 F F F 29 580-599 161 161 161 268 268 268 T T T Max gen: 162, Max flush: 272 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 272 272 272 F F F 1 020-039 162 162 162 272 272 272 F F F 2 040-059 158 158 158 263 263 263 F F F 3 060-079 158 158 158 263 263 263 F F F 4 080-099 161 161 161 268 268 268 F F F 5 100-119 161 161 161 268 268 268 F F F 6 120-139 157 157 157 261 261 261 F F F 7 140-159 158 158 158 263 263 263 F F F 8 160-179 159 159 159 265 265 265 F F F 9 180-199 159 159 159 265 265 265 F F F 10 200-219 163 163 163 273 273 273 F F F 11 220-239 161 161 161 268 268 268 F F F 12 240-259 158 158 158 264 264 264 F F F 13 260-279 162 162 162 271 271 271 F F F 14 280-299 159 159 159 265 265 265 F F F 15 300-319 148 148 148 243 243 243 F F F 16 320-339 161 161 161 268 268 268 F F F 17 340-359 163 163 163 273 273 273 F F F 18 360-379 161 161 161 268 268 268 F F F 19 380-399 163 163 163 273 273 273 F F F 20 400-419 161 161 161 268 268 268 F F F 21 420-439 163 163 163 273 273 273 F F F 22 440-459 161 161 161 268 268 268 F F F 23 460-479 154 154 154 253 253 253 F F F 24 480-499 162 162 162 271 271 271 F F F 25 500-519 161 161 161 268 268 268 F F F 26 520-539 163 163 163 273 273 273 F F F 27 540-559 163 163 163 273 273 273 F F F 28 560-579 162 162 162 271 271 271 F F F 29 580-599 163 163 163 273 273 273 F F F Max gen: 163, Max flush: 273 Begin loop 82 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:07.997965213Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:07.998209579Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:07.998233288Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:07.998258067Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838} {"msg":"Crucible b281344d-6891-47b8-9540-40a9456ed562 has session id: 68c8c64b-a7c2-4094-a188-6fe7d32cf71a","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:07.998433727Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"Upstairs opts: Upstairs UUID: b281344d-6891-47b8-9540-40a9456ed562, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:07.998502433Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"Crucible stats registered with UUID: b281344d-6891-47b8-9540-40a9456ed562","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:07.998533431Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.029622851Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838} {"msg":"b281344d-6891-47b8-9540-40a9456ed562 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.029652429Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.029668038Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"client":"0","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.029692196Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"client":"0","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.029706536Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"client":"1","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.029720455Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"client":"1","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.029734054Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"client":"2","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.029757093Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"client":"2","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.029782542Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.02980757Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.029838408Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"ds_connection connected from Ok(127.0.0.1:35159)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.029972031Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"ds_connection connected from Ok(127.0.0.1:40122)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.030031597Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"ds_connection connected from Ok(127.0.0.1:49959)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.030208077Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.030810073Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"client":"2","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.030843271Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"client":"2","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.030858921Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"client":"1","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.03087267Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"client":"1","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.030886409Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"client":"0","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.030900038Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"client":"0","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.031472446Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"client":"2","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.031502114Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"client":"1","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.031598749Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"client":"0","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.03210245Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"client":"1","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.032141208Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"client":"1","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.032160287Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.032173286Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"client":"0","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.032186885Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"client":"0","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.032203584Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.032216214Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"client":"2","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.032229903Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"client":"2","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"[0]R flush_numbers[0..12]: [272, 272, 263, 263, 268, 268, 261, 263, 265, 265, 273, 268]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.032243872Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"":"downstairs","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"[0]R generation[0..12]: [162, 162, 158, 158, 161, 161, 157, 158, 159, 159, 163, 161]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.032257861Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"":"downstairs","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.032272021Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"":"downstairs","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"[1]R flush_numbers[0..12]: [272, 272, 263, 263, 268, 268, 261, 263, 265, 265, 273, 268]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.0322861Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"":"downstairs","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"[1]R generation[0..12]: [162, 162, 158, 158, 161, 161, 157, 158, 159, 159, 163, 161]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.032300049Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"":"downstairs","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.032314148Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"":"downstairs","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"[2]R flush_numbers[0..12]: [272, 272, 263, 263, 268, 268, 261, 263, 265, 265, 273, 268]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.032328087Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"":"downstairs","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"[2]R generation[0..12]: [162, 162, 158, 158, 161, 161, 157, 158, 159, 159, 163, 161]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.032341757Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"":"downstairs","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.032355916Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"":"downstairs","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.032368925Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"":"downstairs","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.032382154Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"":"downstairs","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"Wait for a query_work_queue command to finish before sending IO msg":"Next flush: 274","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.032405633Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"":"downstairs","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.032431092Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"":"downstairs","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.03245839Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.032482849Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"client":"0","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.032510317Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"client":"1","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.032537486Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"client":"2","":"downstairs","session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.032564634Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.032590063Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"b281344d-6891-47b8-9540-40a9456ed562 is now active with session: 68c8c64b-a7c2-4094-a188-6fe7d32cf71a","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.032616361Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"session_id":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.032648079Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838,"session_id":"68c8c64b-a7c2-4094-a188-6fe7d32cf71a"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:08.032670158Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 518 len 8192 data: 11 12 03/30 Read block 432 len 40960 04/30 Read block 248 len 24576 05/30 Write block 171 len 12288 data: 18 17 15 06/30 Read block 171 len 32768 07/30 Read block 452 len 32768 08/30 Read block 480 len 36864 09/30 Read block 217 len 24576 10/30 Read block 211 len 8192 11/30 Write block 183 len 8192 data: 18 18 12/30 Read block 527 len 16384 13/30 Read block 398 len 12288 14/30 Write block 115 len 8192 data: 15 15 15/30 Write block 573 len 40960 data: 20 17 14 14 17 15 12 8 8 10 16/30 Write block 1 len 4096 data: 9 17/30 Read block 59 len 4096 18/30 Read block 365 len 24576 19/30 Read block 374 len 4096 20/30 Read block 404 len 12288 21/30 Read block 503 len 40960 22/30 Read block 265 len 20480 23/30 Write block 554 len 16384 data: 8 7 9 7 24/30 Flush 25/30 Write block 474 len 28672 data: 10 10 12 10 11 14 13 26/30 Read block 93 len 4096 27/30 Write block 3 len 16384 data: 10 10 9 6 28/30 Read block 351 len 16384 29/30 Write block 212 len 28672 data: 11 12 12 12 16 17 14 30/30 Read block 41 len 32768 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 7 Done Done Done false 26 Acked 1025 Read 1 Done Done Done false 27 Acked 1026 Write 4 Done Done Done false 28 Acked 1027 Read 4 Done Done Done false 29 Acked 1028 Write 7 Done Done Sent false 30 Acked 1029 Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 6 5 4 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:34:08.053699187Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1838} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 275 275 275 T T T 1 020-039 162 162 162 272 272 272 F F F 2 040-059 158 158 158 263 263 263 F F F 3 060-079 158 158 158 263 263 263 F F F 4 080-099 161 161 161 268 268 268 F F F 5 100-119 164 164 164 275 275 275 F F F 6 120-139 157 157 157 261 261 261 F F F 7 140-159 158 158 158 263 263 263 F F F 8 160-179 164 164 164 275 275 275 F F F 9 180-199 164 164 164 275 275 275 F F F 10 200-219 163 163 163 273 273 273 T T T 11 220-239 161 161 161 268 268 268 F F F 12 240-259 158 158 158 264 264 264 F F F 13 260-279 162 162 162 271 271 271 F F F 14 280-299 159 159 159 265 265 265 F F F 15 300-319 148 148 148 243 243 243 F F F 16 320-339 161 161 161 268 268 268 F F F 17 340-359 163 163 163 273 273 273 F F F 18 360-379 161 161 161 268 268 268 F F F 19 380-399 163 163 163 273 273 273 F F F 20 400-419 161 161 161 268 268 268 F F F 21 420-439 163 163 163 273 273 273 F F F 22 440-459 161 161 161 268 268 268 F F F 23 460-479 154 154 154 253 253 253 T T T 24 480-499 162 162 162 271 271 271 T T T 25 500-519 164 164 164 275 275 275 F F F 26 520-539 163 163 163 273 273 273 F F F 27 540-559 164 164 164 275 275 275 F F F 28 560-579 164 164 164 275 275 275 F F F 29 580-599 164 164 164 275 275 275 F F F Max gen: 164, Max flush: 275 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 276 276 276 F F F 1 020-039 162 162 162 272 272 272 F F F 2 040-059 158 158 158 263 263 263 F F F 3 060-079 158 158 158 263 263 263 F F F 4 080-099 161 161 161 268 268 268 F F F 5 100-119 164 164 164 275 275 275 F F F 6 120-139 157 157 157 261 261 261 F F F 7 140-159 158 158 158 263 263 263 F F F 8 160-179 164 164 164 275 275 275 F F F 9 180-199 164 164 164 275 275 275 F F F 10 200-219 165 165 165 276 276 276 F F F 11 220-239 161 161 161 268 268 268 F F F 12 240-259 158 158 158 264 264 264 F F F 13 260-279 162 162 162 271 271 271 F F F 14 280-299 159 159 159 265 265 265 F F F 15 300-319 148 148 148 243 243 243 F F F 16 320-339 161 161 161 268 268 268 F F F 17 340-359 163 163 163 273 273 273 F F F 18 360-379 161 161 161 268 268 268 F F F 19 380-399 163 163 163 273 273 273 F F F 20 400-419 161 161 161 268 268 268 F F F 21 420-439 163 163 163 273 273 273 F F F 22 440-459 161 161 161 268 268 268 F F F 23 460-479 165 165 165 276 276 276 F F F 24 480-499 165 165 165 276 276 276 F F F 25 500-519 164 164 164 275 275 275 F F F 26 520-539 163 163 163 273 273 273 F F F 27 540-559 164 164 164 275 275 275 F F F 28 560-579 164 164 164 275 275 275 F F F 29 580-599 164 164 164 275 275 275 F F F Max gen: 165, Max flush: 276 Begin loop 83 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.426581222Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.426849767Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.426877275Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.426908243Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846} {"msg":"Crucible 5cedc82b-b93e-47f4-9f33-74994857bd55 has session id: 9e4fe0b0-e92f-4cc8-aca7-224196284257","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.427062255Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"Upstairs opts: Upstairs UUID: 5cedc82b-b93e-47f4-9f33-74994857bd55, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.427116292Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"Crucible stats registered with UUID: 5cedc82b-b93e-47f4-9f33-74994857bd55","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.427135481Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.458084748Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846} {"msg":"5cedc82b-b93e-47f4-9f33-74994857bd55 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.458112916Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.458135925Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"client":"0","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.458162364Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"client":"0","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.458179692Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"client":"1","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.458203881Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"client":"1","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.458234089Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"client":"2","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.458265367Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"client":"2","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.458297566Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.458330124Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.458361462Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"ds_connection connected from Ok(127.0.0.1:53435)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.45839291Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"ds_connection connected from Ok(127.0.0.1:40448)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.458430908Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"ds_connection connected from Ok(127.0.0.1:41997)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.45857814Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.459083411Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"client":"1","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.459122579Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"client":"1","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.459152567Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"client":"2","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.459179726Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"client":"2","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.45928168Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"client":"0","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.459311948Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"client":"0","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.45981621Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"client":"1","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.459849028Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"client":"2","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.459877996Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"client":"0","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.460360259Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"client":"1","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.460385787Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"client":"1","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.460400227Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.460475813Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"client":"2","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.460495421Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"client":"2","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.46050894Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.46052155Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"client":"0","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.460534999Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"client":"0","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"[0]R flush_numbers[0..12]: [276, 272, 263, 263, 268, 275, 261, 263, 275, 275, 276, 268]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.460549128Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"":"downstairs","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"[0]R generation[0..12]: [165, 162, 158, 158, 161, 164, 157, 158, 164, 164, 165, 161]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.460563237Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"":"downstairs","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.460582056Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"":"downstairs","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"[1]R flush_numbers[0..12]: [276, 272, 263, 263, 268, 275, 261, 263, 275, 275, 276, 268]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.460596306Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"":"downstairs","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"[1]R generation[0..12]: [165, 162, 158, 158, 161, 164, 157, 158, 164, 164, 165, 161]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.460610005Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"":"downstairs","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.460624114Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"":"downstairs","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"[2]R flush_numbers[0..12]: [276, 272, 263, 263, 268, 275, 261, 263, 275, 275, 276, 268]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.460637733Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"":"downstairs","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"[2]R generation[0..12]: [165, 162, 158, 158, 161, 164, 157, 158, 164, 164, 165, 161]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.460651432Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"":"downstairs","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.460665282Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"":"downstairs","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.460678001Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"":"downstairs","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.46069127Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"":"downstairs","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"Next flush: 277","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.460704089Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"":"downstairs","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.460716859Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"":"downstairs","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.460738627Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.460751147Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"client":"0","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.460764486Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"client":"1","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.460780045Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"client":"2","":"downstairs","session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.460793804Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.460806334Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"5cedc82b-b93e-47f4-9f33-74994857bd55 is now active with session: 9e4fe0b0-e92f-4cc8-aca7-224196284257","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.460819343Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.460831242Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846,"session_id":"9e4fe0b0-e92f-4cc8-aca7-224196284257"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:12.460843501Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 202 len 16384 data: 10 9 12 12 02/30 Write block 331 len 24576 data: 13 11 12 14 12 13 03/30 Read block 41 len 28672 04/30 Read block 504 len 36864 05/30 Read block 17 len 12288 06/30 Write block 442 len 32768 data: 15 16 17 17 15 14 12 12 07/30 Flush 08/30 Write block 189 len 12288 data: 14 13 13 09/30 Write block 71 len 40960 data: 15 16 15 15 11 11 11 13 15 13 10/30 Write block 503 len 40960 data: 20 20 18 16 16 15 12 13 12 13 11/30 Read block 416 len 12288 12/30 Read block 408 len 24576 13/30 Flush 14/30 Write block 259 len 8192 data: 9 10 15/30 Flush 16/30 Write block 174 len 32768 data: 12 12 12 9 9 12 13 11 17/30 Write block 579 len 16384 data: 13 9 9 11 18/30 Flush 19/30 Write block 591 len 36864 data: 11 10 10 11 10 9 8 5 4 20/30 Read block 584 len 32768 21/30 Write block 235 len 20480 data: 5 5 3 4 5 22/30 Read block 472 len 20480 23/30 Write block 69 len 40960 data: 11 13 16 17 16 16 12 12 12 14 24/30 Read block 453 len 36864 25/30 Flush 26/30 Write block 51 len 40960 data: 14 15 13 9 10 11 13 12 12 12 27/30 Read block 218 len 24576 28/30 Write block 355 len 28672 data: 16 15 16 16 16 17 15 29/30 Write block 544 len 24576 data: 14 18 21 20 20 17 30/30 Write block 429 len 12288 data: 10 14 13 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 10 Done Done Done false 27 Acked 1026 Read 6 Done Done Sent false 28 Acked 1027 Write 7 Sent Sent Sent false 29 Acked 1028 Write 6 Sent Sent Sent false 30 Acked 1029 Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 4 10 Done 2 2 1 5 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:34:12.490167651Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1846} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 276 276 276 F F F 1 020-039 162 162 162 272 272 272 F F F 2 040-059 158 158 158 263 263 263 T T T 3 060-079 166 166 166 281 281 281 T T T 4 080-099 166 166 166 278 278 278 F F F 5 100-119 164 164 164 275 275 275 F F F 6 120-139 157 157 157 261 261 261 F F F 7 140-159 158 158 158 263 263 263 F F F 8 160-179 166 166 166 280 280 280 F F F 9 180-199 166 166 166 280 280 280 F F F 10 200-219 166 166 166 277 277 277 F F F 11 220-239 166 166 166 281 281 281 F F F 12 240-259 166 166 166 279 279 279 F F F 13 260-279 166 166 166 279 279 279 F F F 14 280-299 159 159 159 265 265 265 F F F 15 300-319 148 148 148 243 243 243 F F F 16 320-339 166 166 166 277 277 277 F F F 17 340-359 163 163 163 273 273 273 T T T 18 360-379 161 161 161 268 268 268 T T T 19 380-399 163 163 163 273 273 273 F F F 20 400-419 161 161 161 268 268 268 F F F 21 420-439 163 163 163 273 273 273 T T T 22 440-459 166 166 166 277 277 277 F F F 23 460-479 165 165 165 276 276 276 F F F 24 480-499 165 165 165 276 276 276 F F F 25 500-519 166 166 166 278 278 278 F F F 26 520-539 163 163 163 273 273 273 F F F 27 540-559 164 164 164 275 275 275 T T T 28 560-579 166 166 166 280 280 280 F F F 29 580-599 166 166 166 281 281 281 F F F Max gen: 166, Max flush: 281 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 276 276 276 F F F 1 020-039 162 162 162 272 272 272 F F F 2 040-059 167 167 167 282 282 282 F F F 3 060-079 167 167 167 282 282 282 F F F 4 080-099 166 166 166 278 278 278 F F F 5 100-119 164 164 164 275 275 275 F F F 6 120-139 157 157 157 261 261 261 F F F 7 140-159 158 158 158 263 263 263 F F F 8 160-179 166 166 166 280 280 280 F F F 9 180-199 166 166 166 280 280 280 F F F 10 200-219 166 166 166 277 277 277 F F F 11 220-239 166 166 166 281 281 281 F F F 12 240-259 166 166 166 279 279 279 F F F 13 260-279 166 166 166 279 279 279 F F F 14 280-299 159 159 159 265 265 265 F F F 15 300-319 148 148 148 243 243 243 F F F 16 320-339 166 166 166 277 277 277 F F F 17 340-359 167 167 167 282 282 282 F F F 18 360-379 167 167 167 282 282 282 F F F 19 380-399 163 163 163 273 273 273 F F F 20 400-419 161 161 161 268 268 268 F F F 21 420-439 167 167 167 282 282 282 F F F 22 440-459 166 166 166 277 277 277 F F F 23 460-479 165 165 165 276 276 276 F F F 24 480-499 165 165 165 276 276 276 F F F 25 500-519 166 166 166 278 278 278 F F F 26 520-539 163 163 163 273 273 273 F F F 27 540-559 167 167 167 282 282 282 F F F 28 560-579 166 166 166 280 280 280 F F F 29 580-599 166 166 166 281 281 281 F F F Max gen: 167, Max flush: 282 Begin loop 84 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.917112154Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.917394448Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.917415027Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.917441135Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854} {"msg":"Crucible 19165a35-8867-41a2-b82f-2092bf5fd754 has session id: 6ce0b485-5913-4844-967d-5e87869e93f8","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.917564918Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"Upstairs opts: Upstairs UUID: 19165a35-8867-41a2-b82f-2092bf5fd754, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.917617155Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"Crucible stats registered with UUID: 19165a35-8867-41a2-b82f-2092bf5fd754","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.917645293Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.948503906Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854} {"msg":"19165a35-8867-41a2-b82f-2092bf5fd754 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.948534174Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.948551493Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"client":"0","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.948578032Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"client":"0","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.948595371Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"client":"1","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.94861243Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"client":"1","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.948631519Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"client":"2","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.948653768Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"client":"2","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.948683836Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.948715414Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.948745332Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:56132)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.948947921Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:37771)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.948982759Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:33228)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.949012577Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.949429573Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"client":"1","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.949457022Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"client":"1","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.94948115Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"client":"2","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.949540647Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"client":"2","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.949759455Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"client":"0","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.949786433Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"client":"0","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.950265106Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"client":"1","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.950292635Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"client":"2","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.950319383Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"client":"0","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.950824514Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"client":"1","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.950853143Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"client":"1","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.950869252Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.950945918Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"client":"0","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.950967826Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"client":"0","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.950981946Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.950994155Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"client":"2","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.951007574Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"client":"2","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"[0]R flush_numbers[0..12]: [276, 272, 282, 282, 278, 275, 261, 263, 280, 280, 277, 281]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.951021403Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"":"downstairs","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"[0]R generation[0..12]: [165, 162, 167, 167, 166, 164, 157, 158, 166, 166, 166, 166]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.951035482Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"":"downstairs","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.951049632Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"":"downstairs","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"[1]R flush_numbers[0..12]: [276, 272, 282, 282, 278, 275, 261, 263, 280, 280, 277, 281]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.951063681Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"":"downstairs","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"[1]R generation[0..12]: [165, 162, 167, 167, 166, 164, 157, 158, 166, 166, 166, 166]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.95107752Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"":"downstairs","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.951091799Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"":"downstairs","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"[2]R flush_numbers[0..12]: [276, 272, 282, 282, 278, 275, 261, 263, 280, 280, 277, 281]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.951109618Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"":"downstairs","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"[2]R generation[0..12]: [165, 162, 167, 167, 166, 164, 157, 158, 166, 166, 166, 166]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.951123888Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"":"downstairs","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.951137917Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"":"downstairs","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.951150886Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"":"downstairs",Wait for a query_work_queue command to finish before sending IO "":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.951180614Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"":"downstairs","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"Next flush: 283","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.951205233Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"":"downstairs","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.951229882Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"":"downstairs","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.95125344Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.951276139Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"client":"0","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.951293298Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"client":"1","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.951306577Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pidRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":1854,"client":"2","":"downstairs","session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.951324956Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.951337245Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"19165a35-8867-41a2-b82f-2092bf5fd754 is now active with session: 6ce0b485-5913-4844-967d-5e87869e93f8","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.951350345Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.951366724Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854,"session_id":"6ce0b485-5913-4844-967d-5e87869e93f8"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:16.951379333Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 264 len 40960 data: 11 11 12 14 16 15 11 10 10 9 02/30 Flush 03/30 Write block 44 len 4096 data: 8 04/30 Read block 216 len 24576 05/30 Read block 269 len 28672 06/30 Read block 506 len 28672 07/30 Write block 252 len 4096 data: 10 08/30 Read block 227 len 28672 09/30 Read block 68 len 40960 10/30 Read block 427 len 12288 11/30 Write block 470 len 40960 data: 8 11 11 11 11 11 13 11 12 15 12/30 Read block 488 len 8192 13/30 Read block 378 len 32768 14/30 Flush 15/30 Write block 273 len 8192 data: 10 13 16/30 Write block 198 len 20480 data: 12 8 11 9 11 17/30 Flush 18/30 Write block 222 len 28672 data: 12 12 15 14 13 10 11 19/30 Read block 478 len 12288 20/30 Read block 209 len 16384 21/30 Read block 435 len 40960 22/30 Read block 75 len 28672 23/30 Write block 319 len 36864 data: 16 16 14 11 11 11 13 11 14 24/30 Read block 586 len 4096 25/30 Write block 284 len 8192 data: 14 19 26/30 Write block 346 len 16384 data: 10 11 11 10 27/30 Read block 568 len 24576 28/30 Write block 345 len 8192 data: 10 11 29/30 Read block 518 len 20480 30/30 Write block 299 len 24576 data: 13 12 8 14 18 17 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 18 Acked 1017 Write 7 Done Done Done false 19 Acked 1018 Read 3 Done Done Done false 20 Acked 1019 Read 4 Done Done Done false 21 Acked 1020 Read 10 Done Done Done false 22 Acked 1021 Read 7 Done Done Done false 23 Acked 1022 Write 9 Done Done Done false 24 Acked 1023 Read 1 Done Done Done false 25 Acked 1024 Write 2 Done Done Done false 26 Acked 1025 Write 4 Done Done Done false 27 Acked 1026 Read 6 Done Done Done false 28 Acked 1027 Write 2 Done Done Done false 29 Acked 1028 Read 5 Done Done Done false 30 Acked 1029 Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:34:16.97614594Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1854} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 276 276 276 F F F 1 020-039 162 162 162 272 272 272 F F F 2 040-059 168 168 168 284 284 284 F F F 3 060-079 167 167 167 282 282 282 F F F 4 080-099 166 166 166 278 278 278 F F F 5 100-119 164 164 164 275 275 275 F F F 6 120-139 157 157 157 261 261 261 F F F 7 140-159 158 158 158 263 263 263 F F F 8 160-179 166 166 166 280 280 280 F F F 9 180-199 168 168 168 285 285 285 F F F 10 200-219 168 168 168 285 285 285 F F F 11 220-239 166 166 166 281 281 281 T T T 12 240-259 168 168 168 284 284 284 F F F 13 260-279 168 168 168 285 285 285 F F F 14 280-299 159 159 159 265 265 265 T T T 15 300-319 148 148 148 243 243 243 T T T 16 320-339 166 166 166 277 277 277 T T T 17 340-359 167 167 167 282 282 282 T T T 18 360-379 167 167 167 282 282 282 F F F 19 380-399 163 163 163 273 273 273 F F F 20 400-419 161 161 161 268 268 268 F F F 21 420-439 167 167 167 282 282 282 F F F 22 440-459 166 166 166 277 277 277 F F F 23 460-479 168 168 168 284 284 284 F F F 24 480-499 165 165 165 276 276 276 F F F 25 500-519 166 166 166 278 278 278 F F F 26 520-539 163 163 163 273 273 273 F F F 27 540-559 167 167 167 282 282 282 F F F 28 560-579 166 166 166 280 280 280 F F F 29 580-599 166 166 166 281 281 281 F F F Max gen: 168, Max flush: 285 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 276 276 276 F F F 1 020-039 162 162 162 272 272 272 F F F 2 040-059 168 168 168 284 284 284 F F F 3 060-079 167 167 167 282 282 282 F F F 4 080-099 166 166 166 278 278 278 F F F 5 100-119 164 164 164 275 275 275 F F F 6 120-139 157 157 157 261 261 261 F F F 7 140-159 158 158 158 263 263 263 F F F 8 160-179 166 166 166 280 280 280 F F F 9 180-199 168 168 168 285 285 285 F F F 10 200-219 168 168 168 285 285 285 F F F 11 220-239 169 169 169 286 286 286 F F F 12 240-259 168 168 168 284 284 284 F F F 13 260-279 168 168 168 285 285 285 F F F 14 280-299 169 169 169 286 286 286 F F F 15 300-319 169 169 169 286 286 286 F F F 16 320-339 169 169 169 286 286 286 F F F 17 340-359 169 169 169 286 286 286 F F F 18 360-379 167 167 167 282 282 282 F F F 19 380-399 163 163 163 273 273 273 F F F 20 400-419 161 161 161 268 268 268 F F F 21 420-439 167 167 167 282 282 282 F F F 22 440-459 166 166 166 277 277 277 F F F 23 460-479 168 168 168 284 284 284 F F F 24 480-499 165 165 165 276 276 276 F F F 25 500-519 166 166 166 278 278 278 F F F 26 520-539 163 163 163 273 273 273 F F F 27 540-559 167 167 167 282 282 282 F F F 28 560-579 166 166 166 280 280 280 F F F 29 580-599 166 166 166 281 281 281 F F F Max gen: 169, Max flush: 286 Begin loop 85 Downstairs to restart: 2 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.358866548Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.359124543Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.359149772Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.35917925Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862} {"msg":"Crucible a383c1bd-eb20-456c-b6b9-09c7c85abe48 has session id: e070c094-96ff-4161-984d-6a8348a2394b","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.359376959Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"Upstairs opts: Upstairs UUID: a383c1bd-eb20-456c-b6b9-09c7c85abe48, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.359425856Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"Crucible stats registered with UUID: a383c1bd-eb20-456c-b6b9-09c7c85abe48","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.359450165Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.390534905Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862} {"msg":"a383c1bd-eb20-456c-b6b9-09c7c85abe48 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.390567203Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.390582542Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"client":"0","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.39060607Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"client":"0","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.39062047Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"client":"1","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.390633969Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"client":"1","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.390647368Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"client":"2","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.390729094Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"client":"2","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.390762361Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.39078553Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.390814119Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"ds_connection connected from Ok(127.0.0.1:43935)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.390842757Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"ds_connection connected from Ok(127.0.0.1:62107)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.39096247Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"ds_connection connected from Ok(127.0.0.1:60215)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.391069704Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.391611824Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"client":"1","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.391651281Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"client":"1","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.391724857Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"client":"0","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.391759575Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"client":"0","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.391790243Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"client":"2","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.391817502Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"client":"2","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.392283965Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"client":"0","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.392316653Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"client":"1","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.392406819Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"client":"2","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.392811776Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"client":"0","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.392839334Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"client":"0","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.392855913Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.392936698Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"client":"1","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.392957157Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"client":"1","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.392970916Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.393030093Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"client":"2","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.393049792Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"client":"2","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"[0]R flush_numbers[0..12]: [276, 272, 284, 282, 278, 275, 261, 263, 280, 285, 285, 286]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.393064511Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"":"downstairs","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"[0]R generation[0..12]: [165, 162, 168, 167, 166, 164, 157, 158, 166, 168, 168, 169]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.39307869Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"":"downstairs","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.393092829Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"":"downstairs","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"[1]R flush_numbers[0..12]: [276, 272, 284, 282, 278, 275, 261, 263, 280, 285, 285, 286]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.393106689Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"":"downstairs","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"[1]R generation[0..12]: [165, 162, 168, 167, 166, 164, 157, 158, 166, 168, 168, 169]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.393120268Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"":"downstairs","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.393134157Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"":"downstairs","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"[2]R flush_numbers[0..12]: [276, 272, 284, 282, 278, 275, 261, 263, 280, 285, 285, 286]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.393147686Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"":"downstairs","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"[2]R generation[0..12]: [165, 162, 168, 167, 166, 164, 157, 158, 166, 168, 168, 169]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.393161406Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"":"downstairs","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.393175315Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"":"downstairs","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.393191074Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"":"downstairs","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.393204403Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"":"downstairs","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"Next flush: 287","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.393217462Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"":"downstairs","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.393230192Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"":"downstairs","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.393243351Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.39325536Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"client":"0","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.39327163Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"client":"1","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.393285039Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"client":"2","":"downstairs","session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.393298438Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.393310687Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"a383c1bd-eb20-456c-b6b9-09c7c85abe48 is now active with session: e070c094-96ff-4161-984d-6a8348a2394b","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.393323556Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.393335436Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862,"session_id":"e070c094-96ff-4161-984d-6a8348a2394b"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:21.393347515Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 398 len 24576 data: 16 16 14 16 15 14 02/30 Read block 583 len 36864 03/30 Write block 26 len 16384 data: 14 14 17 10 04/30 Write block 290 len 36864 data: 16 15 13 11 10 11 11 9 14 05/30 Write block 443 len 12288 data: 17 18 18 06/30 Write block 253 len 32768 data: 9 11 11 10 6 9 10 11 07/30 Read block 322 len 8192 08/30 Write block 433 len 36864 data: 16 15 17 16 15 14 11 13 13 09/30 Read block 188 len 28672 10/30 Write block 410 len 8192 data: 7 7 11/30 Write block 421 len 40960 data: 13 13 12 11 12 11 13 11 11 15 12/30 Write block 113 len 16384 data: 19 17 16 16 13/30 Write block 42 len 16384 data: 8 9 9 8 14/30 Read block 476 len 36864 15/30 Write block 490 len 40960 data: 14 14 16 15 16 16 16 16 15 17 16/30 Flush 17/30 Write block 299 len 28672 data: 14 13 9 15 19 18 20 18/30 Read block 559 len 28672 19/30 Write block 60 len 32768 data: 13 11 9 11 10 10 13 13 20/30 Flush 21/30 Write block 80 len 36864 data: 14 15 14 12 12 12 16 16 14 22/30 Write block 102 len 8192 data: 14 14 23/30 Read block 484 len 36864 24/30 Read block 453 len 8192 25/30 Write block 220 len 8192 data: 12 11 26/30 Write block 230 len 12288 data: 9 6 6 27/30 Write block 525 len 36864 data: 14 16 16 15 13 13 15 16 17 28/30 Write block 357 len 28672 data: 17 17 17 18 16 16 16 29/30 Read block 196 len 20480 30/30 Write block 504 len 32768 data: 21 19 17 17 16 13 14 13 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 9 Done Done Done false 22 Acked 1021 Write 2 Done Done Done false 23 Acked 1022 Read 9 Done Done Done false 24 Acked 1023 Read 2 Done Done Done false 25 Acked 1024 Write 2 Done Done Done false 26 Acked 1025 Write 3 Done Done Done false 27 Acked 1026 Write 9 Done Done Done false 28 Acked 1027 Write 7 Done Done Done false 29 Acked 1028 Read 5 Done Done Done false 30 Acked 1029 Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:34:21.417849778Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1862} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 276 276 276 F F F 1 020-039 170 170 170 287 287 287 F F F 2 040-059 170 170 170 287 287 287 F F F 3 060-079 170 170 170 288 288 288 F F F 4 080-099 166 166 166 278 278 278 T T T 5 100-119 170 170 170 287 287 287 T T T 6 120-139 157 157 157 261 261 261 F F F 7 140-159 158 158 158 263 263 263 F F F 8 160-179 166 166 166 280 280 280 F F F 9 180-199 168 168 168 285 285 285 F F F 10 200-219 168 168 168 285 285 285 F F F 11 220-239 169 169 169 286 286 286 T T T 12 240-259 170 170 170 287 287 287 F F F 13 260-279 170 170 170 287 287 287 F F F 14 280-299 170 170 170 288 288 288 F F F 15 300-319 170 170 170 288 288 288 F F F 16 320-339 169 169 169 286 286 286 F F F 17 340-359 169 169 169 286 286 286 T T T 18 360-379 167 167 167 282 282 282 T T T 19 380-399 170 170 170 287 287 287 F F F 20 400-419 170 170 170 287 287 287 F F F 21 420-439 170 170 170 287 287 287 F F F 22 440-459 170 170 170 287 287 287 F F F 23 460-479 168 168 168 284 284 284 F F F 24 480-499 170 170 170 287 287 287 F F F 25 500-519 166 166 166 278 278 278 T T T 26 520-539 163 163 163 273 273 273 T T T 27 540-559 167 167 167 282 282 282 F F F 28 560-579 166 166 166 280 280 280 F F F 29 580-599 166 166 166 281 281 281 F F F Max gen: 170, Max flush: 288 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 276 276 276 F F F 1 020-039 170 170 170 287 287 287 F F F 2 040-059 170 170 170 287 287 287 F F F 3 060-079 170 170 170 288 288 288 F F F 4 080-099 171 171 171 289 289 289 F F F 5 100-119 171 171 171 289 289 289 F F F 6 120-139 157 157 157 261 261 261 F F F 7 140-159 158 158 158 263 263 263 F F F 8 160-179 166 166 166 280 280 280 F F F 9 180-199 168 168 168 285 285 285 F F F 10 200-219 168 168 168 285 285 285 F F F 11 220-239 171 171 171 289 289 289 F F F 12 240-259 170 170 170 287 287 287 F F F 13 260-279 170 170 170 287 287 287 F F F 14 280-299 170 170 170 288 288 288 F F F 15 300-319 170 170 170 288 288 288 F F F 16 320-339 169 169 169 286 286 286 F F F 17 340-359 171 171 171 289 289 289 F F F 18 360-379 171 171 171 289 289 289 F F F 19 380-399 170 170 170 287 287 287 F F F 20 400-419 170 170 170 287 287 287 F F F 21 420-439 170 170 170 287 287 287 F F F 22 440-459 170 170 170 287 287 287 F F F 23 460-479 168 168 168 284 284 284 F F F 24 480-499 170 170 170 287 287 287 F F F 25 500-519 171 171 171 289 289 289 F F F 26 520-539 171 171 171 289 289 289 F F F 27 540-559 167 167 167 282 282 282 F F F 28 560-579 166 166 166 280 280 280 F F F 29 580-599 166 166 166 281 281 281 F F F Max gen: 171, Max flush: 289 Begin loop 86 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.794151989Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.79448586Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.794518638Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.794543786Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870} {"msg":"Crucible 683daf17-0375-4ed8-90e3-722eb8d6f8fd has session id: 1a7c28fb-3ec6-437b-88aa-f008b3b07271","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.794629951Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"Upstairs opts: Upstairs UUID: 683daf17-0375-4ed8-90e3-722eb8d6f8fd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.794675159Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"Crucible stats registered with UUID: 683daf17-0375-4ed8-90e3-722eb8d6f8fd","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.794701887Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.825616656Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870} {"msg":"683daf17-0375-4ed8-90e3-722eb8d6f8fd active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.825648485Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.825666974Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"client":"0","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.825694562Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"client":"0","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.825712421Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"client":"1","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.8257298Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"client":"1","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.825746539Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"client":"2","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.825763578Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"client":"2","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.825780417Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.825797916Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.825814905Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"ds_connection connected from Ok(127.0.0.1:38437)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.826049312Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"ds_connection connected from Ok(127.0.0.1:45562)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.82609031Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"ds_connection connected from Ok(127.0.0.1:64500)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.826116568Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.826736303Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"client":"2","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.826771351Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"client":"2","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.82679037Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"client":"0","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.826807469Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"client":"0","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.826910673Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"client":"1","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.826930712Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"client":"1","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.827475481Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"client":"2","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.827514179Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"client":"0","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.827547227Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"client":"1","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.828049899Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"client":"1","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.828075087Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"client":"1","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.828089216Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.828101696Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"client":"2","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.828115235Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"client":"2","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.828128574Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.828144753Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"client":"0","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.828158653Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"client":"0","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"[0]R flush_numbers[0..12]: [276, 287, 287, 288, 289, 289, 261, 263, 280, 285, 285, 289]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.828172452Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"":"downstairs","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"[0]R generation[0..12]: [165, 170, 170, 170, 171, 171, 157, 158, 166, 168, 168, 171]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.828186351Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"":"downstairs","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.82820011Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"":"downstairs","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"[1]R flush_numbers[0..12]: [276, 287, 287, 288, 289, 289, 261, 263, 280, 285, 285, 289]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.8282138Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"":"downstairs","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"[1]R generation[0..12]: [165, 170, 170, 170, 171, 171, 157, 158, 166, 168, 168, 171]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.828227619Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"":"downstairs","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.828241658Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"":"downstairs","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"[2]R flush_numbers[0..12]: [276, 287, 287, 288, 289, 289, 261, 263, 280, 285, 285, 289]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.828255357Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"":"downstairs","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"[2]R generation[0..12]: [165, 170, 170, 170, 171, 171, 157, 158, 166, 168, 168, 171]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.828268966Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"":"downstairs","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.828282836Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"":"downstairs","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.828295805Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"":"downstairs","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.828308824Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"":"downstairs","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"Next flush: 290","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.828321623Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"":"downstairs","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.828336493Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"":"downstairs","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"No downstairs reconciliation required","v":Wait for a query_work_queue command to finish before sending IO 0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.828353401Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.828365621Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"client":"0","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.82837859Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"client":"1","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.82839153Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"client":"2","":"downstairs","session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.828404699Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.828416848Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"683daf17-0375-4ed8-90e3-722eb8d6f8fd is now active with session: 1a7c28fb-3ec6-437b-88aa-f008b3b07271","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.828429607Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.828441487Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870,"session_id":"1a7c28fb-3ec6-437b-88aa-f008b3b07271"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:25.828453566Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 293 len 24576 data: 12 11 12 12 10 15 02/30 Read block 427 len 36864 03/30 Read block 310 len 4096 04/30 Read block 210 len 40960 05/30 Write block 19 len 4096 data: 13 06/30 Read block 389 len 28672 07/30 Write block 68 len 32768 data: 13 12 14 17 18 17 17 13 08/30 Read block 232 len 36864 09/30 Write block 316 len 4096 data: 20 10/30 Write block 59 len 24576 data: 13 14 12 10 12 11 11/30 Read block 127 len 40960 12/30 Write block 569 len 4096 data: 16 13/30 Read block 382 len 8192 14/30 Write block 92 len 32768 data: 16 15 13 11 10 11 9 11 15/30 Read block 338 len 28672 16/30 Read block 26 len 16384 17/30 Write block 467 len 16384 data: 8 7 9 9 18/30 Read block 443 len 8192 19/30 Read block 182 len 4096 20/30 Read block 562 len 36864 21/30 Write block 487 len 12288 data: 11 10 11 22/30 Write block 168 len 12288 data: 13 15 16 23/30 Read block 518 len 12288 24/30 Read block 159 len 24576 25/30 Write block 215 len 40960 data: 13 17 18 15 13 13 12 13 13 16 26/30 Write block 581 len 8192 data: 10 12 27/30 Write block 277 len 4096 data: 17 28/30 Read block 48 len 12288 29/30 Write block 86 len 24576 data: 17 17 15 15 17 16 30/30 Write block 204 len 16384 data: 13 13 13 10 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Write 6 Done Done Done false 2 Acked 1001 Read 9 Done Done Done false 3 Acked 1002 Read 1 Done Done Done false 4 Acked 1003 Read 10 Done Done Done false 5 Acked 1004 Write 1 Done Done Done false 6 Acked 1005 Read 7 Done Done Done false 7 Acked 1006 Write 8 Done Done Done false 8 Acked 1007 Read 9 Done Done Done false 9 Acked 1008 Write 1 Done Done Done false 10 Acked 1009 Write 6 Done Done Done false 11 Acked 1010 Read 10 Done Done Done false 12 Acked 1011 Write 1 Done Done Done false 13 Acked 1012 Read 2 Done Done Done false 14 Acked 1013 Write 8 Done Done Done false 15 Acked 1014 Read 7 Done Done Done false 16 Acked 1015 Read 4 Done Done Done false 17 Acked 1016 Write 4 Done Done Done false 18 Acked 1017 Read 2 Done Done Done false 19 Acked 1018 Read 1 Done Done Done false 20 Acked 1019 Read 9 Done Done Done false 21 Acked 1020 Write 3 Done Done Done false 22 Acked 1021 Write 3 Done Done Done false 23 Acked 1022 Read 3 Done Done Done false 24 Acked 1023 Read 6 Done Done Done false 25 Acked 1024 Write 10 Done Done Done false 26 Acked 1025 Write 2 Done Done Done false 27 Acked 1026 Write 1 Done Done Done false 28 Acked 1027 Read 3 Done Done Done false 29 Acked 1028 Write 6 Sent Sent Sent false 30 Acked 1029 Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:34:25.847068012Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1870} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 276 276 276 T T T 1 020-039 170 170 170 287 287 287 F F F 2 040-059 170 170 170 287 287 287 T T T 3 060-079 170 170 170 288 288 288 T T T 4 080-099 171 171 171 289 289 289 T T T 5 100-119 171 171 171 289 289 289 F F F 6 120-139 157 157 157 261 261 261 F F F 7 140-159 158 158 158 263 263 263 F F F 8 160-179 166 166 166 280 280 280 T T T 9 180-199 168 168 168 285 285 285 F F F 10 200-219 168 168 168 285 285 285 T T T 11 220-239 171 171 171 289 289 289 T T T 12 240-259 170 170 170 287 287 287 F F F 13 260-279 170 170 170 287 287 287 T T T 14 280-299 170 170 170 288 288 288 T T T 15 300-319 170 170 170 288 288 288 T T T 16 320-339 169 169 169 286 286 286 F F F 17 340-359 171 171 171 289 289 289 F F F 18 360-379 171 171 171 289 289 289 F F F 19 380-399 170 170 170 287 287 287 F F F 20 400-419 170 170 170 287 287 287 F F F 21 420-439 170 170 170 287 287 287 F F F 22 440-459 170 170 170 287 287 287 F F F 23 460-479 168 168 168 284 284 284 T T T 24 480-499 170 170 170 287 287 287 T T T 25 500-519 171 171 171 289 289 289 F F F 26 520-539 171 171 171 289 289 289 F F F 27 540-559 167 167 167 282 282 282 F F F 28 560-579 166 166 166 280 280 280 T T T 29 580-599 166 166 166 281 281 281 T T T Max gen: 171, Max flush: 289 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 290 290 290 F F F 1 020-039 170 170 170 287 287 287 F F F 2 040-059 172 172 172 290 290 290 F F F 3 060-079 172 172 172 290 290 290 F F F 4 080-099 172 172 172 290 290 290 F F F 5 100-119 171 171 171 289 289 289 F F F 6 120-139 157 157 157 261 261 261 F F F 7 140-159 158 158 158 263 263 263 F F F 8 160-179 172 172 172 290 290 290 F F F 9 180-199 168 168 168 285 285 285 F F F 10 200-219 172 172 172 290 290 290 F F F 11 220-239 172 172 172 290 290 290 F F F 12 240-259 170 170 170 287 287 287 F F F 13 260-279 172 172 172 290 290 290 F F F 14 280-299 172 172 172 290 290 290 F F F 15 300-319 172 172 172 290 290 290 F F F 16 320-339 169 169 169 286 286 286 F F F 17 340-359 171 171 171 289 289 289 F F F 18 360-379 171 171 171 289 289 289 F F F 19 380-399 170 170 170 287 287 287 F F F 20 400-419 170 170 170 287 287 287 F F F 21 420-439 170 170 170 287 287 287 F F F 22 440-459 170 170 170 287 287 287 F F F 23 460-479 172 172 172 290 290 290 F F F 24 480-499 172 172 172 290 290 290 F F F 25 500-519 171 171 171 289 289 289 F F F 26 520-539 171 171 171 289 289 289 F F F 27 540-559 167 167 167 282 282 282 F F F 28 560-579 172 172 172 290 290 290 F F F 29 580-599 172 172 172 290 290 290 F F F Max gen: 172, Max flush: 290 Begin loop 87 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.350072506Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.35035313Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.350374139Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.350390918Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878} {"msg":"Crucible 7f736a49-e1a1-4971-9bbb-e7395613769a has session id: a11ec7a0-ce6d-442e-9745-655c7e37b300","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.350560778Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"Upstairs opts: Upstairs UUID: 7f736a49-e1a1-4971-9bbb-e7395613769a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.350611656Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"Crucible stats registered with UUID: 7f736a49-e1a1-4971-9bbb-e7395613769a","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.350628924Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.381504806Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878} {"msg":"7f736a49-e1a1-4971-9bbb-e7395613769a active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.381534364Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.381551463Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"client":"0","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.381579532Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"client":"0","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.381597301Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"client":"1","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.38161458Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"client":"1","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.381632519Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"client":"2","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.381649478Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"client":"2","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.381731023Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.381771171Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.381804649Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"ds_connection connected from Ok(127.0.0.1:60845)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.381834517Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"ds_connection connected from Ok(127.0.0.1:46991)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.381868585Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"ds_connection connected from Ok(127.0.0.1:53364)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.381984949Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.382613943Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"client":"1","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.382653271Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"client":"1","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.382682459Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"client":"2","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.382710747Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"client":"2","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.382871658Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"client":"0","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.382913436Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"client":"0","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.383323253Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"client":"1","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.383351521Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"client":"2","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.383671843Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"client":"0","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.383919459Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"client":"2","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.383947417Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"client":"2","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.383963357Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.384022643Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"client":"1","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.384042802Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"client":"1","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.384060781Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.384187474Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"client":"0","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.384206583Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"client":"0","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"[0]R flush_numbers[0..12]: [290, 287, 290, 290, 290, 289, 261, 263, 290, 285, 290, 290]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.384221222Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"":"downstairs","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"[0]R generation[0..12]: [172, 170, 172, 172, 172, 171, 157, 158, 172, 168, 172, 172]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.384235191Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"":"downstairs","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.38424928Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"":"downstairs","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"[1]R flush_numbers[0..12]: [290, 287, 290, 290, 290, 289, 261, 263, 290, 285, 290, 290]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.38426329Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"":"downstairs","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"[1]R generation[0..12]: [172, 170, 172, 172, 172, 171, 157, 158, 172, 168, 172, 172]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.384277039Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"":"downstairs","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.384291018Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"":"downstairs","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"[2]R flush_numbers[0..12]: [290, 287, 290, 290, 290, 289, 261, 263, 290, 285, 290, 290]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.384304637Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"":"downstairs","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"[2]R generation[0..12]: [172, 170, 172, 172, 172, 171, 157, 158, 172, 168, 172, 172]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.384318307Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"":"downstairs","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.384332316Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"":"downstairs","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"Max found gen is 173","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.384345245Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"":"downstairs","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"Generation requested: 174 >= found:173","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.384358504Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"":"downstairs","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"Next flush: 291","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.384373723Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"":"downstairs","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.384386723Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"":"downstairs","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-26T14:34:30.384399832Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.384415221Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"client":"0","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.38442855Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"client":"1","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.38444154Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"client":"2","":"downstairs","session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.384454889Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.384467048Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"7f736a49-e1a1-4971-9bbb-e7395613769a is now active with session: a11ec7a0-ce6d-442e-9745-655c7e37b300","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.384479857Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.384491807Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878,"session_id":"a11ec7a0-ce6d-442e-9745-655c7e37b300"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:30.384503856Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 126 len 40960 data: 12 12 12 12 11 11 11 12 12 11 02/30 Flush 03/30 Write block 545 len 24576 data: 19 22 21 21 18 18 04/30 Read block 237 len 12288 05/30 Read block 101 len 12288 06/30 Write block 255 len 4096 data: 12 07/30 Read block 196 len 40960 08/30 Read block 343 len 20480 09/30 Write block 337 len 12288 data: 9 9 7 10/30 Read block 456 len 36864 11/30 Write block 236 len 12288 data: 6 4 5 12/30 Read block 283 len 36864 13/30 Read block 442 len 4096 14/30 Flush 15/30 Write block 493 len 12288 data: 16 17 17 16/30 Write block 532 len 16384 data: 17 18 13 11 17/30 Read block 70 len 12288 18/30 Flush 19/30 Write block 265 len 24576 data: 12 13 15 17 16 12 20/30 Write block 11 len 36864 data: 10 10 9 8 11 10 10 10 14 21/30 Read block 112 len 4096 22/30 Read block 185 len 36864 23/30 Flush 24/30 Write block 336 len 32768 data: 14 10 10 8 10 9 9 10 25/30 Write block 550 len 20480 data: 19 15 13 12 9 26/30 Read block 97 len 16384 27/30 Read block 251 len 20480 28/30 Read block 414 len 36864 29/30 Read block 492 len 4096 30/30 Read block 351 len 20480 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 8 Done Done Done false 25 Acked 1024 Write 5 Done Done Done false 26 Acked 1025 Read 4 Done Done Done false 27 Acked 1026 Read 5 Done Done Done false 28 Acked 1027 Read 9 Done Done Done false 29 Acked 1028 Read 1 Done Done Done false 30 Acked 1029 Read 5 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 6 6 7 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:34:30.411907664Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1878} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 294 294 294 F F F 1 020-039 170 170 170 287 287 287 F F F 2 040-059 172 172 172 290 290 290 F F F 3 060-079 172 172 172 290 290 290 F F F 4 080-099 172 172 172 290 290 290 F F F 5 100-119 171 171 171 289 289 289 F F F 6 120-139 174 174 174 291 291 291 F F F 7 140-159 158 158 158 263 263 263 F F F 8 160-179 172 172 172 290 290 290 F F F 9 180-199 168 168 168 285 285 285 F F F 10 200-219 172 172 172 290 290 290 F F F 11 220-239 174 174 174 292 292 292 F F F 12 240-259 174 174 174 292 292 292 F F F 13 260-279 174 174 174 294 294 294 F F F 14 280-299 172 172 172 290 290 290 F F F 15 300-319 172 172 172 290 290 290 F F F 16 320-339 174 174 174 292 292 292 T T T 17 340-359 171 171 171 289 289 289 T T T 18 360-379 171 171 171 289 289 289 F F F 19 380-399 170 170 170 287 287 287 F F F 20 400-419 170 170 170 287 287 287 F F F 21 420-439 170 170 170 287 287 287 F F F 22 440-459 170 170 170 287 287 287 F F F 23 460-479 172 172 172 290 290 290 F F F 24 480-499 174 174 174 293 293 293 F F F 25 500-519 171 171 171 289 289 289 F F F 26 520-539 174 174 174 293 293 293 F F F 27 540-559 174 174 174 292 292 292 T T T 28 560-579 172 172 172 290 290 290 F F F 29 580-599 172 172 172 290 290 290 F F F Max gen: 174, Max flush: 294 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 294 294 294 F F F 1 020-039 170 170 170 287 287 287 F F F 2 040-059 172 172 172 290 290 290 F F F 3 060-079 172 172 172 290 290 290 F F F 4 080-099 172 172 172 290 290 290 F F F 5 100-119 171 171 171 289 289 289 F F F 6 120-139 174 174 174 291 291 291 F F F 7 140-159 158 158 158 263 263 263 F F F 8 160-179 172 172 172 290 290 290 F F F 9 180-199 168 168 168 285 285 285 F F F 10 200-219 172 172 172 290 290 290 F F F 11 220-239 174 174 174 292 292 292 F F F 12 240-259 174 174 174 292 292 292 F F F 13 260-279 174 174 174 294 294 294 F F F 14 280-299 172 172 172 290 290 290 F F F 15 300-319 172 172 172 290 290 290 F F F 16 320-339 175 175 175 295 295 295 F F F 17 340-359 175 175 175 295 295 295 F F F 18 360-379 171 171 171 289 289 289 F F F 19 380-399 170 170 170 287 287 287 F F F 20 400-419 170 170 170 287 287 287 F F F 21 420-439 170 170 170 287 287 287 F F F 22 440-459 170 170 170 287 287 287 F F F 23 460-479 172 172 172 290 290 290 F F F 24 480-499 174 174 174 293 293 293 F F F 25 500-519 171 171 171 289 289 289 F F F 26 520-539 174 174 174 293 293 293 F F F 27 540-559 175 175 175 295 295 295 F F F 28 560-579 172 172 172 290 290 290 F F F 29 580-599 172 172 172 290 290 290 F F F Max gen: 175, Max flush: 295 Begin loop 88 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.796461808Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.796727903Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.796745892Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.796761261Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886} {"msg":"Crucible 0825d9ef-5833-474b-b1c7-7344bb1c78cc has session id: 5b243744-5f9b-436b-9e22-59bbf35d8f86","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.797025036Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"Upstairs opts: Upstairs UUID: 0825d9ef-5833-474b-b1c7-7344bb1c78cc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.797073943Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"Crucible stats registered with UUID: 0825d9ef-5833-474b-b1c7-7344bb1c78cc","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.797098352Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.828109056Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886} {"msg":"0825d9ef-5833-474b-b1c7-7344bb1c78cc active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.828142464Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.828159693Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"client":"0","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.828186091Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"client":"0","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.82820347Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"client":"1","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.828223519Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"client":"1","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.828246348Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"client":"2","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.828277846Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"client":"2","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.828308594Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.828340652Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.828374171Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"ds_connection connected from Ok(127.0.0.1:41542)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.828410119Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"ds_connection connected from Ok(127.0.0.1:64105)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.828444817Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"ds_connection connected from Ok(127.0.0.1:45300)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.828580169Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.829209813Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"client":"2","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.829238572Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"client":"2","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.829254421Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"client":"1","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.82927246Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"client":"1","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.829286689Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"client":"0","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.829310048Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"client":"0","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.829870936Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"client":"1","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.830006238Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"client":"0","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.830043216Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"client":"2","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.830422005Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"client":"1","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.830449173Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"client":"1","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.830464102Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.830607294Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"client":"2","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.830628953Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"client":"2","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.830642572Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.830702249Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"client":"0","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.830721588Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"client":"0","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"[0]R flush_numbers[0..12]: [294, 287, 290, 290, 290, 289, 291, 263, 290, 285, 290, 292]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.830738167Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"":"downstairs","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"[0]R generation[0..12]: [174, 170, 172, 172, 172, 171, 174, 158, 172, 168, 172, 174]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.830755516Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"":"downstairs","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.830770145Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"":"downstairs","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"[1]R flush_numbers[0..12]: [294, 287, 290, 290, 290, 289, 291, 263, 290, 285, 290, 292]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.830784184Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"":"downstairs","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"[1]R generation[0..12]: [174, 170, 172, 172, 172, 171, 174, 158, 172, 168, 172, 174]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.830797883Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"":"downstairs","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.830811852Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"":"downstairs","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"[2]R flush_numbers[0..12]: [294, 287, 290, 290, 290, 289, 291, 263, 290, 285, 290, 292]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.830825582Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"":"downstairs","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"[2]R generation[0..12]: [174, 170, 172, 172, 172, 171, 174, 158, 172, 168, 172, 174]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.830839161Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"":"downstairs","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.83085299Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"":"downstairs","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.830866069Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"":"downstairs","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.830879329Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"":"downstairs","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"Next flush: 296","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.830892238Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"":"downstairs","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-26T14:34:34.830905267Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"":"downstairs","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.830918566Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.830931706Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"client":"0","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.830946755Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"client":"1","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.830960154Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"client":"2","":"downstairs","session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.830973313Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.830985693Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"0825d9ef-5833-474b-b1c7-7344bb1c78cc is now active with session: 5b243744-5f9b-436b-9e22-59bbf35d8f86","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.831013841Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.83102813Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886,"session_id":"5b243744-5f9b-436b-9e22-59bbf35d8f86"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:34.83104153Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 321 len 40960 data: 15 12 12 12 14 12 15 15 12 12 02/30 Write block 78 len 4096 data: 15 03/30 Read block 88 len 36864 04/30 Write block 64 len 32768 data: 12 11 14 14 14 13 15 18 05/30 Read block 546 len 24576 06/30 Flush 07/30 Write block 92 len 36864 data: 17 16 14 12 11 12 10 12 12 08/30 Read block 138 len 24576 09/30 Read block 395 len 24576 10/30 Read block 331 len 24576 11/30 Read block 319 len 40960 12/30 Write block 490 len 36864 data: 15 15 17 17 18 18 17 17 16 13/30 Read block 310 len 36864 14/30 Read block 162 len 16384 15/30 Write block 338 len 36864 data: 11 9 11 10 10 11 9 11 12 16/30 Flush 17/30 Write block 391 len 24576 data: 11 11 9 9 10 12 18/30 Flush 19/30 Write block 518 len 16384 data: 12 13 12 10 20/30 Read block 490 len 20480 21/30 Read block 189 len 20480 22/30 Write block 488 len 28672 data: 11 12 16 16 18 18 19 23/30 Read block 90 len 24576 24/30 Read block 333 len 40960 25/30 Read block 155 len 28672 26/30 Flush 27/30 Write block 336 len 20480 data: 15 11 12 10 12 28/30 Read block 162 len 40960 29/30 Write block 90 len 8192 data: 18 17 30/30 Write block 473 len 32768 data: 12 12 12 14 12 13 16 14 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 5 Done Done Done false 28 Acked 1027 Read 10 Done Done Sent false 29 Acked 1028 Write 2 Done Sent Sent false 30 Acked 1029 Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 3 6 Done 3 2 1 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:34:34.859267971Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1886} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 294 294 294 F F F 1 020-039 170 170 170 287 287 287 F F F 2 040-059 172 172 172 290 290 290 F F F 3 060-079 176 176 176 296 296 296 F F F 4 080-099 176 176 176 297 297 297 T T T 5 100-119 176 176 176 297 297 297 F F F 6 120-139 174 174 174 291 291 291 F F F 7 140-159 158 158 158 263 263 263 F F F 8 160-179 172 172 172 290 290 290 F F F 9 180-199 168 168 168 285 285 285 F F F 10 200-219 172 172 172 290 290 290 F F F 11 220-239 174 174 174 292 292 292 F F F 12 240-259 174 174 174 292 292 292 F F F 13 260-279 174 174 174 294 294 294 F F F 14 280-299 172 172 172 290 290 290 F F F 15 300-319 172 172 172 290 290 290 F F F 16 320-339 176 176 176 297 297 297 T T T 17 340-359 176 176 176 297 297 297 T T T 18 360-379 171 171 171 289 289 289 F F F 19 380-399 176 176 176 298 298 298 F F F 20 400-419 170 170 170 287 287 287 F F F 21 420-439 170 170 170 287 287 287 F F F 22 440-459 170 170 170 287 287 287 F F F 23 460-479 172 172 172 290 290 290 T T T 24 480-499 176 176 176 299 299 299 T T T 25 500-519 176 176 176 299 299 299 F F F 26 520-539 176 176 176 299 299 299 F F F 27 540-559 175 175 175 295 295 295 F F F 28 560-579 172 172 172 290 290 290 F F F 29 580-599 172 172 172 290 290 290 F F F Max gen: 176, Max flush: 299 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 294 294 294 F F F 1 020-039 170 170 170 287 287 287 F F F 2 040-059 172 172 172 290 290 290 F F F 3 060-079 176 176 176 296 296 296 F F F 4 080-099 177 177 177 300 300 300 F F F 5 100-119 176 176 176 297 297 297 F F F 6 120-139 174 174 174 291 291 291 F F F 7 140-159 158 158 158 263 263 263 F F F 8 160-179 172 172 172 290 290 290 F F F 9 180-199 168 168 168 285 285 285 F F F 10 200-219 172 172 172 290 290 290 F F F 11 220-239 174 174 174 292 292 292 F F F 12 240-259 174 174 174 292 292 292 F F F 13 260-279 174 174 174 294 294 294 F F F 14 280-299 172 172 172 290 290 290 F F F 15 300-319 172 172 172 290 290 290 F F F 16 320-339 177 177 177 300 300 300 F F F 17 340-359 177 177 177 300 300 300 F F F 18 360-379 171 171 171 289 289 289 F F F 19 380-399 176 176 176 298 298 298 F F F 20 400-419 170 170 170 287 287 287 F F F 21 420-439 170 170 170 287 287 287 F F F 22 440-459 170 170 170 287 287 287 F F F 23 460-479 177 177 177 300 300 300 F F F 24 480-499 177 177 177 300 300 300 F F F 25 500-519 176 176 176 299 299 299 F F F 26 520-539 176 176 176 299 299 299 F F F 27 540-559 175 175 175 295 295 295 F F F 28 560-579 172 172 172 290 290 290 F F F 29 580-599 172 172 172 290 290 290 F F F Max gen: 177, Max flush: 300 Begin loop 89 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.238534284Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.23877828Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.238797819Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.238813298Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894} {"msg":"Crucible e33a84bb-b172-48e7-8426-6b79622b9c05 has session id: 0c668a94-fa51-4372-9469-0f08e79a3e85","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.238980539Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"Upstairs opts: Upstairs UUID: e33a84bb-b172-48e7-8426-6b79622b9c05, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.239098602Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"Crucible stats registered with UUID: e33a84bb-b172-48e7-8426-6b79622b9c05","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.23913084Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.270143364Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894} {"msg":"e33a84bb-b172-48e7-8426-6b79622b9c05 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.270174372Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.270194201Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"client":"0","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.270222299Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"client":"0","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.270240368Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"client":"1","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.270259497Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"client":"1","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.270289256Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"client":"2","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.270327753Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"client":"2","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.270359492Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.27039365Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.270426968Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"ds_connection connected from Ok(127.0.0.1:36779)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.270461876Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"ds_connection connected from Ok(127.0.0.1:51773)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.270632806Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"ds_connection connected from Ok(127.0.0.1:57261)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.270677874Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.271163086Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"client":"2","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.271190545Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"client":"2","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.271335366Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"client":"1","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.271358865Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"client":"1","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.271379664Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"client":"0","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.271393803Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"client":"0","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.271937212Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"client":"2","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.271966781Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"client":"0","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.272018298Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"client":"1","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.272533569Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"client":"2","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.272561087Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"client":"2","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.272576616Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.272648172Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"client":"1","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.272667671Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"client":"1","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.27268127Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.272693699Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"client":"0","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.272707179Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"client":"0","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"[0]R flush_numbers[0..12]: [294, 287, 290, 296, 300, 297, 291, 263, 290, 285, 290, 292]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.272721168Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"":"downstairs","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"[0]R generation[0..12]: [174, 170, 172, 176, 177, 176, 174, 158, 172, 168, 172, 174]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.272735157Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"":"downstairs","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.272749046Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"":"downstairs","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"[1]R flush_numbers[0..12]: [294, 287, 290, 296, 300, 297, 291, 263, 290, 285, 290, 292]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.272762596Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"":"downstairs","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"[1]R generation[0..12]: [174, 170, 172, 176, 177, 176, 174, 158, 172, 168, 172, 174]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.272776195Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"":"downstairs","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.272789874Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"":"downstairs","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"[2]R flush_numbers[0..12]: [294, 287, 290, 296, 300, 297, 291, 263, 290, 285, 290, 292]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.272806763Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"":"downstairs","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"[2]R generation[0..12]: [174, 170, 172, 176, 177, 176, 174, 158, 172, 168, 172, 174]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.272820712Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"":"downstairs","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.272834441Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"":"downstairs","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.272847291Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"":"downstairs","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.27286035Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"":"downstairs","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"Next flush: 301","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.272873069Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"":"downstairs","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.272885909Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"":"downstairs","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.272899048Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.272916657Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"client":"0","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.272930376Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"client":"1","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.272943425Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"client":"2","":"downstairs","session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.272956554Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.272968744Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"e33a84bb-b172-48e7-8426-6b79622b9c05 is now active with session: 0c668a94-fa51-4372-9469-0f08e79a3e85","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.272981473Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.272993472Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894,"session_id":"0c668a94-fa51-4372-9469-0f08e79a3e85"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:39.27302555Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 354 len 28672 data: 16 17 16 18 18 18 19 02/30 Read block 505 len 32768 03/30 Read block 35 len 8192 04/30 Write block 147 len 16384 data: 12 7 9 9 05/30 Read block 66 len 20480 06/30 Read block 383 len 4096 07/30 Read block 291 len 40960 08/30 Write block 252 len 16384 data: 11 10 12 13 09/30 Flush 10/30 Flush 11/30 Write block 452 len 8192 data: 9 9 12/30 Read block 316 len 8192 13/30 Read block 346 len 8192 14/30 Flush 15/30 Write block 298 len 16384 data: 16 15 14 10 16/30 Read block 547 len 24576 17/30 Flush 18/30 Write block 478 len 20480 data: 14 17 15 13 12 19/30 Read block 172 len 8192 20/30 Read block 401 len 36864 21/30 Read block 388 len 20480 22/30 Write block 88 len 24576 data: 16 16 19 18 18 17 23/30 Read block 433 len 36864 24/30 Read block 0 len 8192 25/30 Read block 412 len 20480 26/30 Write block 333 len 12288 data: 13 15 13 27/30 Write block 40 len 20480 data: 7 6 9 10 10 28/30 Read block 309 len 16384 29/30 Write block 105 len 32768 data: 13 10 12 14 13 12 16 17 30/30 Write block 53 len 20480 data: 14 10 11 12 14 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 18 Acked 1017 Write 5 Done Done Done false 19 Acked 1018 Read 2 Done Done Done false 20 Acked 1019 Read 9 Done Done Done false 21 Acked 1020 Read 5 Done Done Done false 22 Acked 1021 Write 6 Done Done Done false 23 Acked 1022 Read 9 Done Done Done false 24 Acked 1023 Read 2 Done Done Done false 25 Acked 1024 Read 5 Done Done Done false 26 Acked 1025 Write 3 Done Done Done false 27 Acked 1026 Write 5 Done Done Done false 28 Acked 1027 Read 4 Done Done Done false 29 Acked 1028 Write 8 Sent Sent Sent false 30 Acked 1029 Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:34:39.297464717Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1894} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 294 294 294 F F F 1 020-039 170 170 170 287 287 287 F F F 2 040-059 172 172 172 290 290 290 T T T 3 060-079 176 176 176 296 296 296 F F F 4 080-099 177 177 177 300 300 300 T T T 5 100-119 176 176 176 297 297 297 T T T 6 120-139 174 174 174 291 291 291 F F F 7 140-159 178 178 178 301 301 301 F F F 8 160-179 172 172 172 290 290 290 F F F 9 180-199 168 168 168 285 285 285 F F F 10 200-219 172 172 172 290 290 290 F F F 11 220-239 174 174 174 292 292 292 F F F 12 240-259 178 178 178 301 301 301 F F F 13 260-279 174 174 174 294 294 294 F F F 14 280-299 178 178 178 304 304 304 F F F 15 300-319 178 178 178 304 304 304 F F F 16 320-339 177 177 177 300 300 300 T T T 17 340-359 178 178 178 301 301 301 F F F 18 360-379 178 178 178 301 301 301 F F F 19 380-399 176 176 176 298 298 298 F F F 20 400-419 170 170 170 287 287 287 F F F 21 420-439 170 170 170 287 287 287 F F F 22 440-459 178 178 178 303 303 303 F F F 23 460-479 177 177 177 300 300 300 T T T 24 480-499 177 177 177 300 300 300 T T T 25 500-519 176 176 176 299 299 299 F F F 26 520-539 176 176 176 299 299 299 F F F 27 540-559 175 175 175 295 295 295 F F F 28 560-579 172 172 172 290 290 290 F F F 29 580-599 172 172 172 290 290 290 F F F Max gen: 178, Max flush: 304 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 294 294 294 F F F 1 020-039 170 170 170 287 287 287 F F F 2 040-059 179 179 179 305 305 305 F F F 3 060-079 176 176 176 296 296 296 F F F 4 080-099 179 179 179 305 305 305 F F F 5 100-119 179 179 179 305 305 305 F F F 6 120-139 174 174 174 291 291 291 F F F 7 140-159 178 178 178 301 301 301 F F F 8 160-179 172 172 172 290 290 290 F F F 9 180-199 168 168 168 285 285 285 F F F 10 200-219 172 172 172 290 290 290 F F F 11 220-239 174 174 174 292 292 292 F F F 12 240-259 178 178 178 301 301 301 F F F 13 260-279 174 174 174 294 294 294 F F F 14 280-299 178 178 178 304 304 304 F F F 15 300-319 178 178 178 304 304 304 F F F 16 320-339 179 179 179 305 305 305 F F F 17 340-359 178 178 178 301 301 301 F F F 18 360-379 178 178 178 301 301 301 F F F 19 380-399 176 176 176 298 298 298 F F F 20 400-419 170 170 170 287 287 287 F F F 21 420-439 170 170 170 287 287 287 F F F 22 440-459 178 178 178 303 303 303 F F F 23 460-479 179 179 179 305 305 305 F F F 24 480-499 179 179 179 305 305 305 F F F 25 500-519 176 176 176 299 299 299 F F F 26 520-539 176 176 176 299 299 299 F F F 27 540-559 175 175 175 295 295 295 F F F 28 560-579 172 172 172 290 290 290 F F F 29 580-599 172 172 172 290 290 290 F F F Max gen: 179, Max flush: 305 Begin loop 90 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.66488747Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.665227741Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.66525487Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.665281888Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902} {"msg":"Crucible bff5ec49-526d-414d-b507-6d3fb9a62282 has session id: 1e30b051-4207-46a6-a739-b86abf0fcdf7","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.665407871Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"Upstairs opts: Upstairs UUID: bff5ec49-526d-414d-b507-6d3fb9a62282, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.665457398Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"Crucible stats registered with UUID: bff5ec49-526d-414d-b507-6d3fb9a62282","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.665481657Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.696929526Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902} {"msg":"bff5ec49-526d-414d-b507-6d3fb9a62282 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.696971243Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.696993672Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"client":"0","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.697047619Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"client":"0","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.697083057Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"client":"1","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.697117755Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"client":"1","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.697152213Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"client":"2","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.697186621Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"client":"2","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.697219529Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.697248128Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.697278086Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"ds_connection connected from Ok(127.0.0.1:32963)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.697310214Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"ds_connection connected from Ok(127.0.0.1:33452)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.697391759Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"ds_connection connected from Ok(127.0.0.1:49576)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.697442087Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.69808761Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"client":"2","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.698121628Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"client":"2","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.698141037Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"client":"1","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.698159136Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"client":"1","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.698176285Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"client":"0","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.698193384Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"client":"0","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.69878839Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"client":"1","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.698836318Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"client":"0","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.698873346Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"client":"2","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.69932695Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"client":"1","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.699354648Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"client":"1","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.699369947Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.699418355Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"client":"0","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.699451283Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"client":"0","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.699474901Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.699488421Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"client":"2","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.69950188Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"client":"2","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"[0]R flush_numbers[0..12]: [294, 287, 305, 296, 305, 305, 291, 301, 290, 285, 290, 292]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.699515839Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"":"downstairs","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"[0]R generation[0..12]: [174, 170, 179, 176, 179, 179, 174, 178, 172, 168, 172, 174]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.699529998Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"":"downstairs","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.699545368Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"":"downstairs","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"[1]R flush_numbers[0..12]: [294, 287, 305, 296, 305, 305, 291, 301, 290, 285, 290, 292]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.699559697Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"":"downstairs","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"[1]R generation[0..12]: [174, 170, 179, 176, 179, 179, 174, 178, 172, 168, 172, 174]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.699573776Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"":"downstairs","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.699587675Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"":"downstairs","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"[2]R flush_numbers[0..12]: [294, 287, 305, 296, 305, 305, 291, 301, 290, 285, 290, 292]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.699601564Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"":"downstairs","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"[2]R generation[0..12]: [174, 170, 179, 176, 179, 179, 174, 178, 172, 168, 172, 174]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.699615253Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"":"downstairs","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.699629363Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"":"downstairs","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.699653551Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"":"downstairs","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-26T14:34:43.69967942Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"":"downstairs","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"Next flush: 306","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.699729887Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"":"downstairs","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.699744836Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"":"downstairs","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.699758485Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.699770475Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"client":"0","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.699784074Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"client":"1","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.699797323Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"client":"2","":"downstairs","session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.699810673Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.699822892Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"bff5ec49-526d-414d-b507-6d3fb9a62282 is now active with session: 1e30b051-4207-46a6-a739-b86abf0fcdf7","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.699835561Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.69984739Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902,"session_id":"1e30b051-4207-46a6-a739-b86abf0fcdf7"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:43.69985992Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 101 len 36864 data: 15 15 15 14 14 11 13 15 14 02/30 Write block 245 len 12288 data: 8 9 12 03/30 Read block 431 len 20480 04/30 Write block 477 len 24576 data: 13 15 18 16 14 13 05/30 Read block 395 len 20480 06/30 Write block 294 len 12288 data: 12 13 13 07/30 Read block 526 len 24576 08/30 Read block 100 len 16384 09/30 Write block 90 len 36864 data: 20 19 19 18 15 13 12 13 11 10/30 Write block 366 len 32768 data: 9 10 9 9 10 11 11 11 11/30 Read block 168 len 28672 12/30 Write block 458 len 28672 data: 10 9 8 10 7 9 9 13/30 Read block 70 len 20480 14/30 Flush 15/30 Write block 288 len 28672 data: 15 17 17 16 14 13 13 16/30 Read block 357 len 20480 17/30 Flush 18/30 Write block 480 len 36864 data: 17 15 14 11 10 11 11 12 12 19/30 Write block 510 len 12288 data: 15 14 14 20/30 Write block 364 len 12288 data: 15 11 10 21/30 Read block 161 len 40960 22/30 Write block 173 len 36864 data: 16 13 13 13 10 10 13 14 12 23/30 Read block 380 len 20480 24/30 Read block 229 len 36864 25/30 Read block 393 len 32768 26/30 Write block 335 len 24576 data: 14 16 12 13 11 13 27/30 Write block 187 len 4096 data: 14 28/30 Read block 559 len 4096 29/30 Read block 239 len 40960 30/30 Write block 155 len 20480 data: 14 19 18 16 16 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 18 Acked 1017 Write 9 Done Done Done false 19 Acked 1018 Write 3 Done Done Done false 20 Acked 1019 Write 3 Done Done Done false 21 Acked 1020 Read 10 Done Done Done false 22 Acked 1021 Write 9 Done Done Done false 23 Acked 1022 Read 5 Done Done Done false 24 Acked 1023 Read 9 Done Done Done false 25 Acked 1024 Read 8 Done Done Done false 26 Acked 1025 Write 6 Done Done Done false 27 Acked 1026 Write 1 Done Done Done false 28 Acked 1027 Read 1 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:34:43.724493305Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1902} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 294 294 294 F F F 1 020-039 170 170 170 287 287 287 F F F 2 040-059 179 179 179 305 305 305 F F F 3 060-079 176 176 176 296 296 296 F F F 4 080-099 180 180 180 306 306 306 F F F 5 100-119 180 180 180 306 306 306 F F F 6 120-139 174 174 174 291 291 291 F F F 7 140-159 178 178 178 301 301 301 T T T 8 160-179 172 172 172 290 290 290 T T T 9 180-199 168 168 168 285 285 285 T T T 10 200-219 172 172 172 290 290 290 F F F 11 220-239 174 174 174 292 292 292 F F F 12 240-259 180 180 180 306 306 306 F F F 13 260-279 174 174 174 294 294 294 F F F 14 280-299 180 180 180 307 307 307 F F F 15 300-319 178 178 178 304 304 304 F F F 16 320-339 179 179 179 305 305 305 T T T 17 340-359 178 178 178 301 301 301 T T T 18 360-379 180 180 180 306 306 306 T T T 19 380-399 176 176 176 298 298 298 F F F 20 400-419 170 170 170 287 287 287 F F F 21 420-439 170 170 170 287 287 287 F F F 22 440-459 180 180 180 306 306 306 F F F 23 460-479 180 180 180 306 306 306 F F F 24 480-499 180 180 180 306 306 306 T T T 25 500-519 176 176 176 299 299 299 T T T 26 520-539 176 176 176 299 299 299 F F F 27 540-559 175 175 175 295 295 295 F F F 28 560-579 172 172 172 290 290 290 F F F 29 580-599 172 172 172 290 290 290 F F F Max gen: 180, Max flush: 307 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 294 294 294 F F F 1 020-039 170 170 170 287 287 287 F F F 2 040-059 179 179 179 305 305 305 F F F 3 060-079 176 176 176 296 296 296 F F F 4 080-099 180 180 180 306 306 306 F F F 5 100-119 180 180 180 306 306 306 F F F 6 120-139 174 174 174 291 291 291 F F F 7 140-159 181 181 181 308 308 308 F F F 8 160-179 181 181 181 308 308 308 F F F 9 180-199 181 181 181 308 308 308 F F F 10 200-219 172 172 172 290 290 290 F F F 11 220-239 174 174 174 292 292 292 F F F 12 240-259 180 180 180 306 306 306 F F F 13 260-279 174 174 174 294 294 294 F F F 14 280-299 180 180 180 307 307 307 F F F 15 300-319 178 178 178 304 304 304 F F F 16 320-339 181 181 181 308 308 308 F F F 17 340-359 181 181 181 308 308 308 F F F 18 360-379 181 181 181 308 308 308 F F F 19 380-399 176 176 176 298 298 298 F F F 20 400-419 170 170 170 287 287 287 F F F 21 420-439 170 170 170 287 287 287 F F F 22 440-459 180 180 180 306 306 306 F F F 23 460-479 180 180 180 306 306 306 F F F 24 480-499 181 181 181 308 308 308 F F F 25 500-519 181 181 181 308 308 308 F F F 26 520-539 176 176 176 299 299 299 F F F 27 540-559 175 175 175 295 295 295 F F F 28 560-579 172 172 172 290 290 290 F F F 29 580-599 172 172 172 290 290 290 F F F Max gen: 181, Max flush: 308 Begin loop 91 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.145251766Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.14570558Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.145767607Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.145800505Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910} {"msg":"Crucible cdd03598-f2ae-47ee-a2dd-5797327a491b has session id: b15e9d48-580e-493e-b27e-76ccbe521262","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.145826263Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"Upstairs opts: Upstairs UUID: cdd03598-f2ae-47ee-a2dd-5797327a491b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.145874351Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"Crucible stats registered with UUID: cdd03598-f2ae-47ee-a2dd-5797327a491b","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.145907289Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.177433733Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910} {"msg":"cdd03598-f2ae-47ee-a2dd-5797327a491b active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.177469971Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.17749336Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"client":"0","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.177536248Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"client":"0","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.177572076Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"client":"1","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.177605324Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"client":"1","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.177637432Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"client":"2","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.17767058Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"client":"2","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.177703728Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.177736646Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.177767874Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"ds_connection connected from Ok(127.0.0.1:42081)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.177800273Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"ds_connection connected from Ok(127.0.0.1:53714)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.177869199Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"ds_connection connected from Ok(127.0.0.1:50744)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.177929165Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.178515022Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"client":"1","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.17854931Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"client":"1","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.178578548Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"client":"0","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.178600517Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"client":"0","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.178667254Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"client":"2","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.178697812Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"client":"2","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.1792557Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"client":"2","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.179295998Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"client":"0","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.179358994Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"client":"1","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.179877265Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"client":"2","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.179902864Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"client":"2","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.179917623Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.180006958Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"client":"1","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.180026726Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"client":"1","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.180040406Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.180056175Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"client":"0","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.180069814Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"client":"0","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"[0]R flush_numbers[0..12]: [294, 287, 305, 296, 306, 306, 291, 308, 308, 308, 290, 292]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.180083523Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"":"downstairs","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"[0]R generation[0..12]: [174, 170, 179, 176, 180, 180, 174, 181, 181, 181, 172, 174]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.180097613Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"":"downstairs","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.180111872Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"":"downstairs","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"[1]R flush_numbers[0..12]: [294, 287, 305, 296, 306, 306, 291, 308, 308, 308, 290, 292]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.180125491Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"":"downstairs","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"[1]R generation[0..12]: [174, 170, 179, 176, 180, 180, 174, 181, 181, 181, 172, 174]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.18013913Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"":"downstairs","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.180153189Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"":"downstairs","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"[2]R flush_numbers[0..12]: [294, 287, 305, 296, 306, 306, 291, 308, 308, 308, 290, 292]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.180167049Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"":"downstairs","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"[2]R generation[0..12]: [174, 170, 179, 176, 180, 180, 174, 181, 181, 181, 172, 174]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.180180598Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"":"downstairs","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.180194547Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"":"downstairs","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.180207386Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"":"downstairs","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.180220455Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"":"downstairs","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"Next flush: 309","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.180233275Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"":"downstairs","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.180248044Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"":"downstairs","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.180261483Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"session_idWait for a query_work_queue command to finish before sending IO ":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.180277372Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"client":"0","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.180290732Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"client":"1","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.180303781Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"client":"2","":"downstairs","session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.18031716Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.180329309Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"cdd03598-f2ae-47ee-a2dd-5797327a491b is now active with session: b15e9d48-580e-493e-b27e-76ccbe521262","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.180341839Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.180353758Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910,"session_id":"b15e9d48-580e-493e-b27e-76ccbe521262"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:48.180365967Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 321 len 40960 data: 16 13 13 13 15 13 16 16 13 13 02/30 Read block 367 len 28672 03/30 Read block 86 len 32768 04/30 Read block 504 len 32768 05/30 Read block 47 len 8192 06/30 Read block 99 len 40960 07/30 Read block 534 len 32768 08/30 Flush 09/30 Write block 582 len 20480 data: 13 12 13 15 15 10/30 Write block 321 len 40960 data: 17 14 14 14 16 14 17 17 14 14 11/30 Write block 115 len 28672 data: 17 17 14 13 15 15 19 12/30 Write block 221 len 20480 data: 13 14 14 17 15 13/30 Read block 407 len 12288 14/30 Read block 399 len 8192 15/30 Read block 393 len 8192 16/30 Write block 48 len 24576 data: 15 14 17 15 16 15 17/30 Write block 110 len 4096 data: 13 18/30 Read block 424 len 8192 19/30 Write block 378 len 24576 data: 12 10 11 9 10 11 20/30 Read block 572 len 24576 21/30 Write block 303 len 24576 data: 20 19 21 15 14 14 22/30 Flush 23/30 Write block 332 len 40960 data: 12 14 16 15 17 13 14 12 14 11 24/30 Write block 199 len 16384 data: 9 12 10 12 25/30 Write block 17 len 32768 data: 11 11 15 11 11 14 16 14 26/30 Write block 425 len 24576 data: 13 12 14 12 12 16 27/30 Read block 8 len 24576 28/30 Write block 443 len 32768 data: 18 19 19 16 15 13 13 11 29/30 Write block 581 len 12288 data: 11 14 13 30/30 Read block 53 len 8192 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 10 Done Done Done false 24 Acked 1023 Write 4 Done Done Done false 25 Acked 1024 Write 8 Done Done Done false 26 Acked 1025 Write 6 Done Done Done false 27 Acked 1026 Read 6 Done Done Done false 28 Acked 1027 Write 8 Done Done Sent false 29 Acked 1028 Write 3 Done Done Sent false 30 Acked 1029 Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 3 4 Done 8 7 5 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:34:48.20363478Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1910} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 294 294 294 T T T 1 020-039 170 170 170 287 287 287 T T T 2 040-059 182 182 182 310 310 310 F F F 3 060-079 176 176 176 296 296 296 F F F 4 080-099 180 180 180 306 306 306 F F F 5 100-119 182 182 182 310 310 310 F F F 6 120-139 182 182 182 310 310 310 F F F 7 140-159 181 181 181 308 308 308 F F F 8 160-179 181 181 181 308 308 308 F F F 9 180-199 181 181 181 308 308 308 T T T 10 200-219 172 172 172 290 290 290 T T T 11 220-239 182 182 182 310 310 310 F F F 12 240-259 180 180 180 306 306 306 F F F 13 260-279 174 174 174 294 294 294 F F F 14 280-299 180 180 180 307 307 307 F F F 15 300-319 182 182 182 310 310 310 F F F 16 320-339 182 182 182 310 310 310 T T T 17 340-359 181 181 181 308 308 308 T T T 18 360-379 182 182 182 310 310 310 F F F 19 380-399 182 182 182 310 310 310 F F F 20 400-419 170 170 170 287 287 287 F F F 21 420-439 170 170 170 287 287 287 T T T 22 440-459 180 180 180 306 306 306 T T T 23 460-479 180 180 180 306 306 306 F F F 24 480-499 181 181 181 308 308 308 F F F 25 500-519 181 181 181 308 308 308 F F F 26 520-539 176 176 176 299 299 299 F F F 27 540-559 175 175 175 295 295 295 F F F 28 560-579 172 172 172 290 290 290 F F F 29 580-599 182 182 182 310 310 310 T T T Max gen: 182, Max flush: 310 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 311 311 311 F F F 1 020-039 183 183 183 311 311 311 F F F 2 040-059 182 182 182 310 310 310 F F F 3 060-079 176 176 176 296 296 296 F F F 4 080-099 180 180 180 306 306 306 F F F 5 100-119 182 182 182 310 310 310 F F F 6 120-139 182 182 182 310 310 310 F F F 7 140-159 181 181 181 308 308 308 F F F 8 160-179 181 181 181 308 308 308 F F F 9 180-199 183 183 183 311 311 311 F F F 10 200-219 183 183 183 311 311 311 F F F 11 220-239 182 182 182 310 310 310 F F F 12 240-259 180 180 180 306 306 306 F F F 13 260-279 174 174 174 294 294 294 F F F 14 280-299 180 180 180 307 307 307 F F F 15 300-319 182 182 182 310 310 310 F F F 16 320-339 183 183 183 311 311 311 F F F 17 340-359 183 183 183 311 311 311 F F F 18 360-379 182 182 182 310 310 310 F F F 19 380-399 182 182 182 310 310 310 F F F 20 400-419 170 170 170 287 287 287 F F F 21 420-439 183 183 183 311 311 311 F F F 22 440-459 183 183 183 311 311 311 F F F 23 460-479 180 180 180 306 306 306 F F F 24 480-499 181 181 181 308 308 308 F F F 25 500-519 181 181 181 308 308 308 F F F 26 520-539 176 176 176 299 299 299 F F F 27 540-559 175 175 175 295 295 295 F F F 28 560-579 172 172 172 290 290 290 F F F 29 580-599 183 183 183 311 311 311 F F F Max gen: 183, Max flush: 311 Begin loop 92 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.643089464Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.64334729Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.643366779Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.643385078Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918} {"msg":"Crucible 677d9b3c-9a51-4439-a9e1-93983c5bd6d1 has session id: 8c7a70c4-885b-47a3-b180-90f7e4f2aa6f","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.643650872Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"Upstairs opts: Upstairs UUID: 677d9b3c-9a51-4439-a9e1-93983c5bd6d1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.6436999Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"Crucible stats registered with UUID: 677d9b3c-9a51-4439-a9e1-93983c5bd6d1","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.643723608Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.675542636Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918} {"msg":"677d9b3c-9a51-4439-a9e1-93983c5bd6d1 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.675573085Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.675595593Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"client":"0","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.675625462Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"client":"0","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.675643641Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"client":"1","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.67566113Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"client":"1","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.675677819Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"client":"2","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.675700167Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"client":"2","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.675729786Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.675759224Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.675785763Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"ds_connection connected from Ok(127.0.0.1:43464)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.675813501Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"ds_connection connected from Ok(127.0.0.1:61320)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.675859498Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"ds_connection connected from Ok(127.0.0.1:56905)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.676038238Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.676662363Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"client":"0","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.676698411Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"client":"0","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.67671786Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"client":"2","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.676735519Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"client":"2","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.676759857Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"client":"1","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.676781676Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"client":"1","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.677250709Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"client":"0","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.677353344Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"client":"1","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.677389182Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"client":"2","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.677796689Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"client":"0","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.677824747Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"client":"0","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.677839716Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.677925411Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"client":"1","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.67794761Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"client":"1","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.677961169Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.677973339Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"client":"2","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.677986688Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"client":"2","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"[0]R flush_numbers[0..12]: [311, 311, 310, 296, 306, 310, 310, 308, 308, 311, 311, 310]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.678000637Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"":"downstairs","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"[0]R generation[0..12]: [183, 183, 182, 176, 180, 182, 182, 181, 181, 183, 183, 182]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.678014566Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"":"downstairs","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.678032275Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"":"downstairs","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"[1]R flush_numbers[0..12]: [311, 311, 310, 296, 306, 310, 310, 308, 308, 311, 311, 310]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.678046624Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"":"downstairs","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"[1]R generation[0..12]: [183, 183, 182, 176, 180, 182, 182, 181, 181, 183, 183, 182]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.678060344Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"":"downstairs","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.678074353Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"":"downstairs","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"[2]R flush_numbers[0..12]: [311, 311, 310, 296, 306, 310, 310, 308, 308, 311, 311, 310]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.678088212Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"":"downstairs","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"[2]R generation[0..12]: [183, 183, 182, 176, 180, 182, 182, 181, 181, 183, 183, 182]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.678102221Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"":"downstairs","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.67811606Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"":"downstairs","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.678129Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"":"downstairs","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.678142409Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"":"downstairs","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"Next flush: 312","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-26T14:34:52.678155288Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"":"downstairs","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.678168268Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"":"downstairs","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.678181797Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.678193746Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"client":"0","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.678206746Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"client":"1","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.678221794Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"client":"2","":"downstairs","session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.678235134Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.678247293Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"677d9b3c-9a51-4439-a9e1-93983c5bd6d1 is now active with session: 8c7a70c4-885b-47a3-b180-90f7e4f2aa6f","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.678260003Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.678271912Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918,"session_id":"8c7a70c4-885b-47a3-b180-90f7e4f2aa6f"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:52.678284061Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 467 len 4096 data: 9 02/30 Read block 306 len 40960 03/30 Read block 250 len 12288 04/30 Read block 495 len 32768 05/30 Read block 564 len 36864 06/30 Read block 378 len 40960 07/30 Read block 65 len 12288 08/30 Write block 110 len 20480 data: 14 17 18 20 18 09/30 Write block 483 len 16384 data: 12 11 12 12 10/30 Write block 425 len 4096 data: 14 11/30 Write block 224 len 8192 data: 18 16 12/30 Write block 294 len 4096 data: 14 13/30 Read block 258 len 36864 14/30 Write block 443 len 32768 data: 19 20 20 17 16 14 14 12 15/30 Write block 212 len 32768 data: 12 13 13 14 18 19 16 14 16/30 Write block 567 len 20480 data: 12 16 17 17 20 17/30 Write block 482 len 4096 data: 15 18/30 Write block 560 len 40960 data: 10 9 9 10 8 11 10 13 17 18 19/30 Read block 571 len 32768 20/30 Read block 159 len 32768 21/30 Write block 165 len 16384 data: 10 9 14 14 22/30 Write block 40 len 16384 data: 8 7 10 11 23/30 Flush 24/30 Write block 238 len 16384 data: 6 6 5 8 25/30 Read block 371 len 28672 26/30 Read block 307 len 12288 27/30 Write block 412 len 24576 data: 9 12 13 12 13 12 28/30 Read block 438 len 16384 29/30 Write block 544 len 16384 data: 15 20 23 22 30/30 Write block 507 len 20480 data: 18 17 14 16 15 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 4 Done Done Done false 25 Acked 1024 Read 7 Done Done Done false 26 Acked 1025 Read 3 Done Done Done false 27 Acked 1026 Write 6 Done Done Done false 28 Acked 1027 Read 4 Done Done Done false 29 Acked 1028 Write 4 Sent Sent Sent false 30 Acked 1029 Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:34:52.699252974Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1918} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 311 311 311 F F F 1 020-039 183 183 183 311 311 311 F F F 2 040-059 184 184 184 312 312 312 F F F 3 060-079 176 176 176 296 296 296 F F F 4 080-099 180 180 180 306 306 306 F F F 5 100-119 184 184 184 312 312 312 F F F 6 120-139 182 182 182 310 310 310 F F F 7 140-159 181 181 181 308 308 308 F F F 8 160-179 184 184 184 312 312 312 F F F 9 180-199 183 183 183 311 311 311 F F F 10 200-219 184 184 184 312 312 312 F F F 11 220-239 184 184 184 312 312 312 T T T 12 240-259 180 180 180 306 306 306 T T T 13 260-279 174 174 174 294 294 294 F F F 14 280-299 184 184 184 312 312 312 F F F 15 300-319 182 182 182 310 310 310 F F F 16 320-339 183 183 183 311 311 311 F F F 17 340-359 183 183 183 311 311 311 F F F 18 360-379 182 182 182 310 310 310 F F F 19 380-399 182 182 182 310 310 310 F F F 20 400-419 170 170 170 287 287 287 T T T 21 420-439 184 184 184 312 312 312 F F F 22 440-459 184 184 184 312 312 312 F F F 23 460-479 184 184 184 312 312 312 F F F 24 480-499 184 184 184 312 312 312 F F F 25 500-519 181 181 181 308 308 308 T T T 26 520-539 176 176 176 299 299 299 F F F 27 540-559 175 175 175 295 295 295 T T T 28 560-579 184 184 184 312 312 312 F F F 29 580-599 183 183 183 311 311 311 F F F Max gen: 184, Max flush: 312 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 311 311 311 F F F 1 020-039 183 183 183 311 311 311 F F F 2 040-059 184 184 184 312 312 312 F F F 3 060-079 176 176 176 296 296 296 F F F 4 080-099 180 180 180 306 306 306 F F F 5 100-119 184 184 184 312 312 312 F F F 6 120-139 182 182 182 310 310 310 F F F 7 140-159 181 181 181 308 308 308 F F F 8 160-179 184 184 184 312 312 312 F F F 9 180-199 183 183 183 311 311 311 F F F 10 200-219 184 184 184 312 312 312 F F F 11 220-239 185 185 185 313 313 313 F F F 12 240-259 185 185 185 313 313 313 F F F 13 260-279 174 174 174 294 294 294 F F F 14 280-299 184 184 184 312 312 312 F F F 15 300-319 182 182 182 310 310 310 F F F 16 320-339 183 183 183 311 311 311 F F F 17 340-359 183 183 183 311 311 311 F F F 18 360-379 182 182 182 310 310 310 F F F 19 380-399 182 182 182 310 310 310 F F F 20 400-419 185 185 185 313 313 313 F F F 21 420-439 184 184 184 312 312 312 F F F 22 440-459 184 184 184 312 312 312 F F F 23 460-479 184 184 184 312 312 312 F F F 24 480-499 184 184 184 312 312 312 F F F 25 500-519 185 185 185 313 313 313 F F F 26 520-539 176 176 176 299 299 299 F F F 27 540-559 185 185 185 313 313 313 F F F 28 560-579 184 184 184 312 312 312 F F F 29 580-599 183 183 183 311 311 311 F F F Max gen: 185, Max flush: 313 Begin loop 93 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.048008644Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.048347785Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.048377363Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.048397482Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926} {"msg":"Crucible b4f27b49-6a7e-4af6-a310-ad9a309e0898 has session id: 22d8b3cd-d7f4-4412-9f37-c1b03f92a03b","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.048586541Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"Upstairs opts: Upstairs UUID: b4f27b49-6a7e-4af6-a310-ad9a309e0898, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.048631749Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"Crucible stats registered with UUID: b4f27b49-6a7e-4af6-a310-ad9a309e0898","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.048654948Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.07950805Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926} {"msg":"b4f27b49-6a7e-4af6-a310-ad9a309e0898 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.079544878Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.079564437Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"client":"0","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.079588846Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"client":"0","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.079606335Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"client":"1","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.079623224Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"client":"1","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.079640633Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"client":"2","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.079674361Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"client":"2","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.079705749Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.079735987Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.079767366Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"ds_connection connected from Ok(127.0.0.1:42627)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.079802304Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"ds_connection connected from Ok(127.0.0.1:40020)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.079836012Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"ds_connection connected from Ok(127.0.0.1:58913)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.080012942Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.080622857Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"client":"2","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.080655765Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"client":"2","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.080687083Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"client":"1","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.080763819Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"client":"1","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.080795337Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"client":"0","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.080822596Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"client":"0","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.081313958Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"client":"2","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.081343196Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"client":"0","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.081484418Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"client":"1","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.081945842Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"client":"2","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.081973191Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"client":"2","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.08198762Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.082000399Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"client":"0","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.082013958Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"client":"0","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.082027828Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.082040267Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"client":"1","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.082054006Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"client":"1","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"[0]R flush_numbers[0..12]: [311, 311, 312, 296, 306, 312, 310, 308, 312, 311, 312, 313]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.082068165Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"":"downstairs","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"[0]R generation[0..12]: [183, 183, 184, 176, 180, 184, 182, 181, 184, 183, 184, 185]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.082082435Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"":"downstairs","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.082096834Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"":"downstairs","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"[1]R flush_numbers[0..12]: [311, 311, 312, 296, 306, 312, 310, 308, 312, 311, 312, 313]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.082110863Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"":"downstairs","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"[1]R generation[0..12]: [183, 183, 184, 176, 180, 184, 182, 181, 184, 183, 184, 185]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.082124602Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"":"downstairs","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.082138651Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"":"downstairs","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"[2]R flush_numbers[0..12]: [311, 311, 312, 296, 306, 312, 310, 308, 312, 311, 312, 313]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.08215687Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"":"downstairs","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"[2]R generation[0..12]: [183, 183, 184, 176, 180, 184, 182, 181, 184, 183, 184, 185]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.08217123Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"":"downstairs","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.082185179Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"":"downstairs","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.082198278Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"":"downstairs","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-26T14:34:57.082211547Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"":"downstairs","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"Next flush: 314","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.082228306Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"":"downstairs","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.082241466Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"":"downstairs","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.082254805Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.082266774Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"client":"0","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.082280183Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"client":"1","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.082293143Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"client":"2","":"downstairs","session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.082306452Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.082318681Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"b4f27b49-6a7e-4af6-a310-ad9a309e0898 is now active with session: 22d8b3cd-d7f4-4412-9f37-c1b03f92a03b","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.08233167Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.08234366Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926,"session_id":"22d8b3cd-d7f4-4412-9f37-c1b03f92a03b"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:34:57.082357849Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 274 len 32768 data: 14 15 18 18 18 17 16 14 02/30 Write block 525 len 16384 data: 15 17 17 16 03/30 Read block 29 len 4096 04/30 Read block 496 len 40960 05/30 Read block 597 len 8192 06/30 Read block 207 len 40960 07/30 Write block 585 len 8192 data: 16 16 08/30 Write block 209 len 36864 data: 11 12 11 13 14 14 15 19 20 09/30 Write block 508 len 36864 data: 18 15 17 16 15 11 12 10 10 10/30 Flush 11/30 Write block 192 len 40960 data: 15 11 10 11 12 10 13 10 13 11 12/30 Flush 13/30 Write block 102 len 4096 data: 16 14/30 Read block 19 len 4096 15/30 Write block 538 len 8192 data: 14 16 16/30 Write block 561 len 20480 data: 10 10 11 9 12 17/30 Write block 550 len 12288 data: 20 16 14 18/30 Flush 19/30 Write block 544 len 8192 data: 16 21 20/30 Read block 416 len 16384 21/30 Write block 202 len 20480 data: 13 10 14 14 14 22/30 Write block 413 len 8192 data: 13 14 23/30 Read block 200 len 36864 24/30 Write block 308 len 12288 data: 15 12 8 25/30 Write block 372 len 32768 data: 12 12 12 12 10 11 13 11 26/30 Write block 156 len 40960 data: 20 19 17 17 13 11 10 9 9 11 27/30 Read block 393 len 36864 28/30 Read block 273 len 24576 29/30 Read block 382 len 20480 30/30 Write block 477 len 8192 data: 14 16 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 19 Acked 1018 Write 2 Done Done Done false 20 Acked 1019 Read 4 Done Done Done false 21 Acked 1020 Write 5 Done Done Done false 22 Acked 1021 Write 2 Done Done Done false 23 Acked 1022 Read 9 Done Done Done false 24 Acked 1023 Write 3 Done Done Done false 25 Acked 1024 Write 8 Done Done Done false 26 Acked 1025 Write 10 Done Done Done false 27 Acked 1026 Read 9 Done Done Done false 28 Acked 1027 Read 6 Done Done Done false 29 Acked 1028 Read 5 Done Done Done false 30 Acked 1029 Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:34:57.106042298Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1926} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 311 311 311 F F F 1 020-039 183 183 183 311 311 311 F F F 2 040-059 184 184 184 312 312 312 F F F 3 060-079 176 176 176 296 296 296 F F F 4 080-099 180 180 180 306 306 306 F F F 5 100-119 186 186 186 316 316 316 F F F 6 120-139 182 182 182 310 310 310 F F F 7 140-159 181 181 181 308 308 308 T T T 8 160-179 184 184 184 312 312 312 T T T 9 180-199 186 186 186 315 315 315 F F F 10 200-219 186 186 186 315 315 315 T T T 11 220-239 185 185 185 313 313 313 F F F 12 240-259 185 185 185 313 313 313 F F F 13 260-279 186 186 186 314 314 314 F F F 14 280-299 186 186 186 314 314 314 F F F 15 300-319 182 182 182 310 310 310 T T T 16 320-339 183 183 183 311 311 311 F F F 17 340-359 183 183 183 311 311 311 F F F 18 360-379 182 182 182 310 310 310 T T T 19 380-399 182 182 182 310 310 310 F F F 20 400-419 185 185 185 313 313 313 T T T 21 420-439 184 184 184 312 312 312 F F F 22 440-459 184 184 184 312 312 312 F F F 23 460-479 184 184 184 312 312 312 T T T 24 480-499 184 184 184 312 312 312 F F F 25 500-519 186 186 186 314 314 314 F F F 26 520-539 186 186 186 316 316 316 F F F 27 540-559 186 186 186 316 316 316 T T T 28 560-579 186 186 186 316 316 316 F F F 29 580-599 186 186 186 314 314 314 F F F Max gen: 186, Max flush: 316 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 311 311 311 F F F 1 020-039 183 183 183 311 311 311 F F F 2 040-059 184 184 184 312 312 312 F F F 3 060-079 176 176 176 296 296 296 F F F 4 080-099 180 180 180 306 306 306 F F F 5 100-119 186 186 186 316 316 316 F F F 6 120-139 182 182 182 310 310 310 F F F 7 140-159 187 187 187 317 317 317 F F F 8 160-179 187 187 187 317 317 317 F F F 9 180-199 186 186 186 315 315 315 F F F 10 200-219 187 187 187 317 317 317 F F F 11 220-239 185 185 185 313 313 313 F F F 12 240-259 185 185 185 313 313 313 F F F 13 260-279 186 186 186 314 314 314 F F F 14 280-299 186 186 186 314 314 314 F F F 15 300-319 187 187 187 317 317 317 F F F 16 320-339 183 183 183 311 311 311 F F F 17 340-359 183 183 183 311 311 311 F F F 18 360-379 187 187 187 317 317 317 F F F 19 380-399 182 182 182 310 310 310 F F F 20 400-419 187 187 187 317 317 317 F F F 21 420-439 184 184 184 312 312 312 F F F 22 440-459 184 184 184 312 312 312 F F F 23 460-479 187 187 187 317 317 317 F F F 24 480-499 184 184 184 312 312 312 F F F 25 500-519 186 186 186 314 314 314 F F F 26 520-539 186 186 186 316 316 316 F F F 27 540-559 187 187 187 317 317 317 F F F 28 560-579 186 186 186 316 316 316 F F F 29 580-599 186 186 186 314 314 314 F F F Max gen: 187, Max flush: 317 Begin loop 94 Downstairs to restart: 2 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.497159859Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.497460852Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.497488951Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.497518149Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934} {"msg":"Crucible fe70a95c-6c3d-4252-8f62-7d739a70acb4 has session id: 6da00c97-a329-4365-9260-ae9b3e065202","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.497624103Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"Upstairs opts: Upstairs UUID: fe70a95c-6c3d-4252-8f62-7d739a70acb4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.49766786Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"Crucible stats registered with UUID: fe70a95c-6c3d-4252-8f62-7d739a70acb4","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.497692969Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.528818196Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934} {"msg":"fe70a95c-6c3d-4252-8f62-7d739a70acb4 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.528854284Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.528875883Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"client":"0","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.528919Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"client":"0","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.528955058Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"client":"1","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.528990176Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"client":"1","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.529023334Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"client":"2","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.529053573Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"client":"2","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.529071482Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.52909465Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.529126339Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"ds_connection connected from Ok(127.0.0.1:52519)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.529163706Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"ds_connection connected from Ok(127.0.0.1:46470)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.529387614Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"ds_connection connected from Ok(127.0.0.1:62485)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.529419422Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.529829749Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"client":"1","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.529857157Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"client":"1","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.529872346Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"client":"0","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.529886056Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"client":"0","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.529899405Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"client":"2","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.529961112Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"client":"2","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.530556308Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"client":"1","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.530581476Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"client":"2","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.530607485Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"client":"0","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.531157814Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"client":"1","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.531186162Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"client":"1","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.531201531Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.53121808Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"client":"2","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.531232149Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"client":"2","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.531245509Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.531257778Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"client":"0","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.531271017Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"client":"0","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"[0]R flush_numbers[0..12]: [311, 311, 312, 296, 306, 316, 310, 317, 317, 315, 317, 313]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.531284817Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"":"downstairs","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"[0]R generation[0..12]: [183, 183, 184, 176, 180, 186, 182, 187, 187, 186, 187, 185]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.531298726Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"":"downstairs","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.531312925Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"":"downstairs","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"[1]R flush_numbers[0..12]: [311, 311, 312, 296, 306, 316, 310, 317, 317, 315, 317, 313]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.531326984Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"":"downstairs","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"[1]R generation[0..12]: [183, 183, 184, 176, 180, 186, 182, 187, 187, 186, 187, 185]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.531340843Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"":"downstairs","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.531354773Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"":"downstairs","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"[2]R flush_numbers[0..12]: [311, 311, 312, 296, 306, 316, 310, 317, 317, 315, 317, 313]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.531368562Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"":"downstairs","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"[2]R generation[0..12]: [183, 183, 184, 176, 180, 186, 182, 187, 187, 186, 187, 185]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.531382381Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"":"downstairs","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.53139699Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"":"downstairs","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.531412209Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"":"downstairs","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-26T14:35:01.531425809Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"":"downstairs","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"Next flush: 318","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.531441058Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"":"downstairs","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.531454117Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"":"downstairs","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.531467536Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.531479535Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"client":"0","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.531492815Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"client":"1","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.531505964Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"client":"2","":"downstairs","session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.531519053Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.531531452Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"fe70a95c-6c3d-4252-8f62-7d739a70acb4 is now active with session: 6da00c97-a329-4365-9260-ae9b3e065202","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.531544522Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934,"session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.531556541Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ","pid":1934,"session_id":"6da00c97-a329-4365-9260-ae9b3e065202"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:01.5315744Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Flush 04/30 Write block 594 len 16384 data: 12 11 10 9 05/30 Write block 121 len 40960 data: 20 17 15 18 18 13 13 13 13 12 06/30 Read block 116 len 4096 07/30 Flush 08/30 Write block 459 len 4096 data: 10 09/30 Write block 8 len 32768 data: 7 9 11 11 11 10 9 12 10/30 Read block 465 len 8192 11/30 Read block 276 len 28672 12/30 Write block 285 len 20480 data: 20 16 15 16 18 13/30 Read block 530 len 40960 14/30 Read block 500 len 24576 15/30 Write block 139 len 4096 data: 11 16/30 Read block 505 len 4096 17/30 Write block 324 len 36864 data: 15 17 15 18 18 15 15 14 13 18/30 Write block 86 len 32768 data: 18 18 17 17 21 20 20 19 19/30 Read block 415 len 36864 20/30 Write block 413 len 16384 data: 14 15 13 14 21/30 Read block 416 len 28672 22/30 Write block 255 len 40960 data: 14 11 7 10 11 12 11 11 11 12 23/30 Write block 49 len 16384 data: 15 18 16 17 24/30 Read block 153 len 28672 25/30 Read block 51 len 40960 26/30 Read block 413 len 32768 27/30 Write block 34 len 32768 data: 6 9 8 7 8 8 9 8 28/30 Read block 95 len 36864 29/30 Write block 538 len 28672 data: 15 17 16 16 17 18 17 30/30 Read block 37 len 36864 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 8 Acked 1007 Write 1 Done Done Done false 9 Acked 1008 Write 8 Done Done Done false 10 Acked 1009 Read 2 Done Done Done false 11 Acked 1010 Read 7 Done Done Done false 12 Acked 1011 Write 5 Done Done Done false 13 Acked 1012 Read 10 Done Done Done false 14 Acked 1013 Read 6 Done Done Done false 15 Acked 1014 Write 1 Done Done Done false 16 Acked 1015 Read 1 Done Done Done false 17 Acked 1016 Write 9 Done Done Done false 18 Acked 1017 Write 8 Done Done Done false 19 Acked 1018 Read 9 Done Done Done false 20 Acked 1019 Write 4 Done Done Done false 21 Acked 1020 Read 7 Done Done Done false 22 Acked 1021 Write 10 Done Done Done false 23 Acked 1022 Write 4 Done Done Done false 24 Acked 1023 Read 7 Done Done Done false 25 Acked 1024 Read 10 Done Done Done false 26 Acked 1025 Read 8 Done Done Done false 27 Acked 1026 Write 8 Done Done Done false 28 Acked 1027 Read 9 Done Done Done false 29 Acked 1028 Write 7 Done Sent Done false 30 Acked 1029 Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 0 3 Done 22 21 23 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1006 1006 1006 Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:35:01.553720056Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1934} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 311 311 311 T T T 1 020-039 183 183 183 311 311 311 T T T 2 040-059 184 184 184 312 312 312 T T T 3 060-079 176 176 176 296 296 296 F F F 4 080-099 180 180 180 306 306 306 T T T 5 100-119 186 186 186 316 316 316 F F F 6 120-139 188 188 188 321 321 321 T T T 7 140-159 187 187 187 317 317 317 F F F 8 160-179 187 187 187 317 317 317 F F F 9 180-199 186 186 186 315 315 315 F F F 10 200-219 187 187 187 317 317 317 F F F 11 220-239 185 185 185 313 313 313 F F F 12 240-259 185 185 185 313 313 313 T T T 13 260-279 186 186 186 314 314 314 T T T 14 280-299 186 186 186 314 314 314 T T T 15 300-319 187 187 187 317 317 317 F F F 16 320-339 183 183 183 311 311 311 T T T 17 340-359 183 183 183 311 311 311 F F F 18 360-379 187 187 187 317 317 317 F F F 19 380-399 182 182 182 310 310 310 F F F 20 400-419 187 187 187 317 317 317 T T T 21 420-439 184 184 184 312 312 312 F F F 22 440-459 184 184 184 312 312 312 T T T 23 460-479 187 187 187 317 317 317 F F F 24 480-499 184 184 184 312 312 312 F F F 25 500-519 186 186 186 314 314 314 F F F 26 520-539 186 186 186 316 316 316 T T T 27 540-559 187 187 187 317 317 317 T T T 28 560-579 186 186 186 316 316 316 F F F 29 580-599 188 188 188 321 321 321 F F F Max gen: 188, Max flush: 321 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 322 322 322 F F F 1 020-039 189 189 189 322 322 322 F F F 2 040-059 189 189 189 322 322 322 F F F 3 060-079 176 176 176 296 296 296 F F F 4 080-099 189 189 189 322 322 322 F F F 5 100-119 186 186 186 316 316 316 F F F 6 120-139 189 189 189 322 322 322 F F F 7 140-159 187 187 187 317 317 317 F F F 8 160-179 187 187 187 317 317 317 F F F 9 180-199 186 186 186 315 315 315 F F F 10 200-219 187 187 187 317 317 317 F F F 11 220-239 185 185 185 313 313 313 F F F 12 240-259 189 189 189 322 322 322 F F F 13 260-279 189 189 189 322 322 322 F F F 14 280-299 189 189 189 322 322 322 F F F 15 300-319 187 187 187 317 317 317 F F F 16 320-339 189 189 189 322 322 322 F F F 17 340-359 183 183 183 311 311 311 F F F 18 360-379 187 187 187 317 317 317 F F F 19 380-399 182 182 182 310 310 310 F F F 20 400-419 189 189 189 322 322 322 F F F 21 420-439 184 184 184 312 312 312 F F F 22 440-459 189 189 189 322 322 322 F F F 23 460-479 187 187 187 317 317 317 F F F 24 480-499 184 184 184 312 312 312 F F F 25 500-519 186 186 186 314 314 314 F F F 26 520-539 189 189 189 322 322 322 F F F 27 540-559 189 189 189 322 322 322 F F F 28 560-579 186 186 186 316 316 316 F F F 29 580-599 188 188 188 321 321 321 F F F Max gen: 189, Max flush: 322 Begin loop 95 Downstairs to restart: 2 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.012422579Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.01275545Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.012789038Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.012815516Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942} {"msg":"Crucible c93b000f-8045-4447-bc98-ea5558819c2e has session id: 9da9c38a-76c5-438d-a23a-9fedf849ac54","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.013039034Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"Upstairs opts: Upstairs UUID: c93b000f-8045-4447-bc98-ea5558819c2e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.013082011Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"Crucible stats registered with UUID: c93b000f-8045-4447-bc98-ea5558819c2e","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.0130998Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.044210158Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942} {"msg":"c93b000f-8045-4447-bc98-ea5558819c2e active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.044242627Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.044262046Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"client":"0","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.044290964Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"client":"0","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.044308963Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"client":"1","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.044332222Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"client":"1","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.0443568Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"client":"2","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.044387248Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"client":"2","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.044417627Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.044448415Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.044479683Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"ds_connection connected from Ok(127.0.0.1:56872)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.044514071Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"ds_connection connected from Ok(127.0.0.1:62685)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.044550699Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"ds_connection connected from Ok(127.0.0.1:38634)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.044581527Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.045341874Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"client":"0","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.045370373Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"client":"0","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.045386562Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"client":"1","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.04541112Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"client":"1","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.045479737Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"client":"2","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.045504665Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"client":"2","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.046046274Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"client":"2","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.046086002Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"client":"1","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.04611716Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"client":"0","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.046624142Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"client":"2","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.04664921Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"client":"2","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.046663599Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.046744075Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"client":"1","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.046780103Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"client":"1","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.046802551Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.046815541Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"client":"0","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.04682908Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"client":"0","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"[0]R flush_numbers[0..12]: [322, 322, 322, 296, 322, 316, 322, 317, 317, 315, 317, 313]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.046843229Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"":"downstairs","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"[0]R generation[0..12]: [189, 189, 189, 176, 189, 186, 189, 187, 187, 186, 187, 185]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.046857448Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"":"downstairs","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.046871608Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"":"downstairs","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"[1]R flush_numbers[0..12]: [322, 322, 322, 296, 322, 316, 322, 317, 317, 315, 317, 313]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.046885687Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"":"downstairs","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"[1]R generation[0..12]: [189, 189, 189, 176, 189, 186, 189, 187, 187, 186, 187, 185]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.046899526Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"":"downstairs","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.046913675Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"":"downstairs","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"[2]R flush_numbers[0..12]: [322, 322, 322, 296, 322, 316, 322, 317, 317, 315, 317, 313]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.046927635Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"":"downstairs","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"[2]R generation[0..12]: [189, 189, 189, 176, 189, 186, 189, 187, 187, 186, 187, 185]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.046941584Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"":"downstairs","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.046961833Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"":"downstairs","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"timeWait for a query_work_queue command to finish before sending IO ":"2024-09-26T14:35:06.046986501Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"":"downstairs","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.047018919Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"":"downstairs","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"Next flush: 323","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.047034698Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"":"downstairs","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.047047908Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"":"downstairs","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.047060977Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.047073026Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"client":"0","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.047086215Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"client":"1","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.047099175Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"client":"2","":"downstairs","session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.047112394Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.047124923Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"c93b000f-8045-4447-bc98-ea5558819c2e is now active with session: 9da9c38a-76c5-438d-a23a-9fedf849ac54","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.047137723Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.047149662Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942,"session_id":"9da9c38a-76c5-438d-a23a-9fedf849ac54"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:06.047161921Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 187 len 8192 data: 15 17 02/30 Write block 180 len 24576 data: 15 13 15 19 19 17 03/30 Write block 508 len 12288 data: 19 16 18 04/30 Write block 450 len 32768 data: 13 10 10 10 11 12 13 13 05/30 Flush 06/30 Write block 349 len 8192 data: 11 12 07/30 Write block 116 len 16384 data: 18 15 14 16 08/30 Write block 435 len 20480 data: 18 17 16 15 12 09/30 Read block 582 len 8192 10/30 Write block 579 len 24576 data: 14 10 12 15 14 14 11/30 Read block 483 len 4096 12/30 Read block 3 len 20480 13/30 Read block 522 len 28672 14/30 Write block 273 len 40960 data: 11 15 16 19 19 19 18 17 15 17 15/30 Read block 405 len 4096 16/30 Write block 403 len 24576 data: 15 11 9 9 5 6 17/30 Write block 561 len 40960 data: 11 11 12 10 13 11 14 18 19 18 18/30 Read block 256 len 12288 19/30 Read block 534 len 20480 20/30 Read block 184 len 28672 21/30 Write block 278 len 32768 data: 20 19 18 16 18 15 15 21 22/30 Write block 500 len 20480 data: 15 17 19 21 22 23/30 Read block 400 len 24576 24/30 Read block 510 len 16384 25/30 Read block 217 len 8192 26/30 Read block 162 len 4096 27/30 Read block 365 len 32768 28/30 Write block 475 len 36864 data: 13 15 15 17 19 18 16 16 13 29/30 Read block 535 len 40960 30/30 Write block 251 len 32768 data: 9 12 11 13 15 12 8 11 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 6 Acked 1005 Write 2 Done Done Done false 7 Acked 1006 Write 4 Done Done Done false 8 Acked 1007 Write 5 Done Done Done false 9 Acked 1008 Read 2 Done Done Done false 10 Acked 1009 Write 6 Done Done Done false 11 Acked 1010 Read 1 Done Done Done false 12 Acked 1011 Read 5 Done Done Done false 13 Acked 1012 Read 7 Done Done Done false 14 Acked 1013 Write 10 Done Done Done false 15 Acked 1014 Read 1 Done Done Done false 16 Acked 1015 Write 6 Done Done Done false 17 Acked 1016 Write 10 Done Done Done false 18 Acked 1017 Read 3 Done Done Done false 19 Acked 1018 Read 5 Done Done Done false 20 Acked 1019 Read 7 Done Done Done false 21 Acked 1020 Write 8 Done Done Done false 22 Acked 1021 Write 5 Done Done Done false 23 Acked 1022 Read 6 Done Done Done false 24 Acked 1023 Read 4 Done Done Done false 25 Acked 1024 Read 2 Done Done Done false 26 Acked 1025 Read 1 Done Done Done false 27 Acked 1026 Read 8 Done Done Done false 28 Acked 1027 Write 9 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 24 24 24 72 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1004 1004 1004 Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:35:06.068824494Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1942} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 322 322 322 F F F 1 020-039 189 189 189 322 322 322 F F F 2 040-059 189 189 189 322 322 322 F F F 3 060-079 176 176 176 296 296 296 F F F 4 080-099 189 189 189 322 322 322 F F F 5 100-119 186 186 186 316 316 316 T T T 6 120-139 189 189 189 322 322 322 F F F 7 140-159 187 187 187 317 317 317 F F F 8 160-179 187 187 187 317 317 317 F F F 9 180-199 190 190 190 323 323 323 F F F 10 200-219 187 187 187 317 317 317 F F F 11 220-239 185 185 185 313 313 313 F F F 12 240-259 189 189 189 322 322 322 T T T 13 260-279 189 189 189 322 322 322 T T T 14 280-299 189 189 189 322 322 322 T T T 15 300-319 187 187 187 317 317 317 F F F 16 320-339 189 189 189 322 322 322 F F F 17 340-359 183 183 183 311 311 311 T T T 18 360-379 187 187 187 317 317 317 F F F 19 380-399 182 182 182 310 310 310 F F F 20 400-419 189 189 189 322 322 322 T T T 21 420-439 184 184 184 312 312 312 T T T 22 440-459 190 190 190 323 323 323 F F F 23 460-479 187 187 187 317 317 317 T T T 24 480-499 184 184 184 312 312 312 T T T 25 500-519 190 190 190 323 323 323 T T T 26 520-539 189 189 189 322 322 322 F F F 27 540-559 189 189 189 322 322 322 F F F 28 560-579 186 186 186 316 316 316 T T T 29 580-599 188 188 188 321 321 321 T T T Max gen: 190, Max flush: 323 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 322 322 322 F F F 1 020-039 189 189 189 322 322 322 F F F 2 040-059 189 189 189 322 322 322 F F F 3 060-079 176 176 176 296 296 296 F F F 4 080-099 189 189 189 322 322 322 F F F 5 100-119 191 191 191 324 324 324 F F F 6 120-139 189 189 189 322 322 322 F F F 7 140-159 187 187 187 317 317 317 F F F 8 160-179 187 187 187 317 317 317 F F F 9 180-199 190 190 190 323 323 323 F F F 10 200-219 187 187 187 317 317 317 F F F 11 220-239 185 185 185 313 313 313 F F F 12 240-259 191 191 191 324 324 324 F F F 13 260-279 191 191 191 324 324 324 F F F 14 280-299 191 191 191 324 324 324 F F F 15 300-319 187 187 187 317 317 317 F F F 16 320-339 189 189 189 322 322 322 F F F 17 340-359 191 191 191 324 324 324 F F F 18 360-379 187 187 187 317 317 317 F F F 19 380-399 182 182 182 310 310 310 F F F 20 400-419 191 191 191 324 324 324 F F F 21 420-439 191 191 191 324 324 324 F F F 22 440-459 190 190 190 323 323 323 F F F 23 460-479 191 191 191 324 324 324 F F F 24 480-499 191 191 191 324 324 324 F F F 25 500-519 191 191 191 324 324 324 F F F 26 520-539 189 189 189 322 322 322 F F F 27 540-559 189 189 189 322 322 322 F F F 28 560-579 191 191 191 324 324 324 F F F 29 580-599 191 191 191 324 324 324 F F F Max gen: 191, Max flush: 324 Begin loop 96 Downstairs to restart: 0 look at: http://127.0.0.1:12810 Failed to get info from "http://127.0.0.1:12810" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146) look at: http://127.0.0.1:12820 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.549019832Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.549336894Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.549367242Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.549392501Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950} {"msg":"Crucible 38669e70-2c22-4d44-8975-2188bbeff320 has session id: d321060d-ff37-4374-aac2-f53d0f2d3713","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.549487235Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"Upstairs opts: Upstairs UUID: 38669e70-2c22-4d44-8975-2188bbeff320, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.549531063Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"Crucible stats registered with UUID: 38669e70-2c22-4d44-8975-2188bbeff320","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.549549822Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.580744915Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950} {"msg":"38669e70-2c22-4d44-8975-2188bbeff320 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.580776073Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.580794412Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"client":"0","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.580823181Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"client":"0","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.58084111Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"client":"1","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.580861068Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"client":"1","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.580877818Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"client":"2","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.580894897Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"client":"2","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.580911546Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"":"io task","client":"2","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.580929035Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.580946394Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"ds_connection connected from Ok(127.0.0.1:38721)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.581082806Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"":"io task","client":"2","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"ds_connection connected from Ok(127.0.0.1:63412)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.581130553Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"ds_connection connected from Ok(127.0.0.1:41708)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.58119534Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.58188673Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"client":"1","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.581927178Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"client":"1","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.581956486Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"client":"2","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.581987425Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"client":"2","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.582017153Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"client":"0","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.582045092Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"client":"0","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.582473227Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"client":"1","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.582592521Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"client":"0","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.582612479Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"client":"2","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.583100352Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"client":"1","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.58312849Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"client":"1","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.583143549Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.583169628Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"client":"2","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.583186607Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"client":"2","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.583200366Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.583274702Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"client":"0","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.583294721Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"client":"0","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"[0]R flush_numbers[0..12]: [322, 322, 322, 296, 322, 324, 322, 317, 317, 323, 317, 313]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.58330919Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"":"downstairs","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"[0]R generation[0..12]: [189, 189, 189, 176, 189, 191, 189, 187, 187, 190, 187, 185]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.583323369Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"":"downstairs","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.583337498Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"":"downstairs","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"[1]R flush_numbers[0..12]: [322, 322, 322, 296, 322, 324, 322, 317, 317, 323, 317, 313]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.583351218Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"":"downstairs","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"[1]R generation[0..12]: [189, 189, 189, 176, 189, 191, 189, 187, 187, 190, 187, 185]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.583364917Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"":"downstairs","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.583378796Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"":"downstairs","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"[2]R flush_numbers[0..12]: [322, 322, 322, 296, 322, 324, 322, 317, 317, 323, 317, 313]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.583392925Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"":"downstairs","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"[2]R generation[0..12]: [189, 189, 189, 176, 189, 191, 189, 187, 187, 190, 187, 185]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.583406515Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"":"downstairs","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.583420464Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"":"downstairs","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.583433153Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"":"downstairs","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.583449032Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"":"downstairs","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"Next flush: 325","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.583462221Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"":"downstairs","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.583475171Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"":"downstairs","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-26T14:35:10.58348839Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.583502319Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"client":"0","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.583515438Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"client":"1","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.583528517Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"client":"2","":"downstairs","session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.583541607Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.583553786Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"38669e70-2c22-4d44-8975-2188bbeff320 is now active with session: d321060d-ff37-4374-aac2-f53d0f2d3713","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.583566485Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.583578405Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950,"session_id":"d321060d-ff37-4374-aac2-f53d0f2d3713"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:10.583590584Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 151 len 28672 data: 11 10 11 11 15 21 20 02/30 Read block 369 len 12288 03/30 Read block 407 len 4096 04/30 Read block 202 len 24576 05/30 Read block 94 len 40960 06/30 Read block 505 len 16384 07/30 Flush 08/30 Write block 471 len 40960 data: 12 12 13 13 14 16 16 18 20 19 09/30 Read block 438 len 20480 10/30 Write block 449 len 36864 data: 15 14 11 11 11 12 13 14 14 11/30 Read block 423 len 32768 12/30 Flush 13/30 Write block 437 len 36864 data: 17 16 13 14 14 16 20 21 21 14/30 Write block 379 len 36864 data: 12 12 10 11 12 9 9 9 10 15/30 Write block 191 len 28672 data: 14 16 12 11 12 13 11 16/30 Read block 307 len 40960 17/30 Read block 481 len 12288 18/30 Read block 494 len 40960 19/30 Flush 20/30 Write block 331 len 32768 data: 15 14 15 17 16 18 14 15 21/30 Read block 154 len 16384 22/30 Read block 109 len 8192 23/30 Write block 226 len 8192 data: 14 11 24/30 Write block 583 len 40960 data: 15 15 17 17 15 14 14 14 12 11 25/30 Read block 181 len 20480 26/30 Read block 531 len 40960 27/30 Read block 369 len 8192 28/30 Write block 202 len 28672 data: 14 11 15 15 15 11 7 29/30 Write block 160 len 8192 data: 14 12 30/30 Read block 396 len 28672 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 20 Acked 1019 Write 8 Done Done Done false 21 Acked 1020 Read 4 Done Done Done false 22 Acked 1021 Read 2 Done Done Done false 23 Acked 1022 Write 2 Done Done Done false 24 Acked 1023 Write 10 Done Done Done false 25 Acked 1024 Read 5 Done Done Done false 26 Acked 1025 Read 10 Done Done Done false 27 Acked 1026 Read 2 Done Done Done false 28 Acked 1027 Write 7 Done Done Done false 29 Acked 1028 Write 2 Done Done Done false 30 Acked 1029 Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 10 11 10 31 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:35:10.610733777Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1950} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 322 322 322 F F F 1 020-039 189 189 189 322 322 322 F F F 2 040-059 189 189 189 322 322 322 F F F 3 060-079 176 176 176 296 296 296 F F F 4 080-099 189 189 189 322 322 322 F F F 5 100-119 191 191 191 324 324 324 F F F 6 120-139 189 189 189 322 322 322 F F F 7 140-159 192 192 192 325 325 325 F F F 8 160-179 187 187 187 317 317 317 T T T 9 180-199 192 192 192 327 327 327 F F F 10 200-219 187 187 187 317 317 317 T T T 11 220-239 185 185 185 313 313 313 T T T 12 240-259 191 191 191 324 324 324 F F F 13 260-279 191 191 191 324 324 324 F F F 14 280-299 191 191 191 324 324 324 F F F 15 300-319 187 187 187 317 317 317 F F F 16 320-339 189 189 189 322 322 322 T T T 17 340-359 191 191 191 324 324 324 F F F 18 360-379 192 192 192 327 327 327 F F F 19 380-399 192 192 192 327 327 327 F F F 20 400-419 191 191 191 324 324 324 F F F 21 420-439 192 192 192 327 327 327 F F F 22 440-459 192 192 192 327 327 327 F F F 23 460-479 192 192 192 326 326 326 F F F 24 480-499 192 192 192 326 326 326 F F F 25 500-519 191 191 191 324 324 324 F F F 26 520-539 189 189 189 322 322 322 F F F 27 540-559 189 189 189 322 322 322 F F F 28 560-579 191 191 191 324 324 324 F F F 29 580-599 191 191 191 324 324 324 T T T Max gen: 192, Max flush: 327 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 322 322 322 F F F 1 020-039 189 189 189 322 322 322 F F F 2 040-059 189 189 189 322 322 322 F F F 3 060-079 176 176 176 296 296 296 F F F 4 080-099 189 189 189 322 322 322 F F F 5 100-119 191 191 191 324 324 324 F F F 6 120-139 189 189 189 322 322 322 F F F 7 140-159 192 192 192 325 325 325 F F F 8 160-179 193 193 193 328 328 328 F F F 9 180-199 192 192 192 327 327 327 F F F 10 200-219 193 193 193 328 328 328 F F F 11 220-239 193 193 193 328 328 328 F F F 12 240-259 191 191 191 324 324 324 F F F 13 260-279 191 191 191 324 324 324 F F F 14 280-299 191 191 191 324 324 324 F F F 15 300-319 187 187 187 317 317 317 F F F 16 320-339 193 193 193 328 328 328 F F F 17 340-359 191 191 191 324 324 324 F F F 18 360-379 192 192 192 327 327 327 F F F 19 380-399 192 192 192 327 327 327 F F F 20 400-419 191 191 191 324 324 324 F F F 21 420-439 192 192 192 327 327 327 F F F 22 440-459 192 192 192 327 327 327 F F F 23 460-479 192 192 192 326 326 326 F F F 24 480-499 192 192 192 326 326 326 F F F 25 500-519 191 191 191 324 324 324 F F F 26 520-539 189 189 189 322 322 322 F F F 27 540-559 189 189 189 322 322 322 F F F 28 560-579 191 191 191 324 324 324 F F F 29 580-599 193 193 193 328 328 328 F F F Max gen: 193, Max flush: 328 Begin loop 97 Downstairs to restart: 1 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:14.99705496Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:14.997298836Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:14.997316415Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:14.997331834Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958} {"msg":"Crucible 5a237da8-3039-49df-a446-80485a192238 has session id: 85c0f243-8d5f-4978-93e0-a2579e0060b9","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:14.9975826Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"Upstairs opts: Upstairs UUID: 5a237da8-3039-49df-a446-80485a192238, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:14.997634047Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"Crucible stats registered with UUID: 5a237da8-3039-49df-a446-80485a192238","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:14.997659936Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.029381989Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958} {"msg":"5a237da8-3039-49df-a446-80485a192238 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.029415637Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.029431077Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"client":"0","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.029451535Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"client":"0","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.029465915Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"client":"1","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.029480044Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"client":"1","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.029493623Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"client":"2","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.029507242Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"client":"2","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.029520541Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"":"io task","client":"2","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.02953416Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"":"io task","client":"0","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.029560389Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:53883)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.029694682Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"":"io task","client":"0","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:60299)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.029730129Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:54781)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.029756438Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"":"io task","client":"2","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.030553833Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"client":"2","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.030585101Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"client":"2","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.03060036Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"client":"0","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.030614539Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"client":"0","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.030631149Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"client":"1","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.030656447Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"client":"1","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.031265922Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"client":"1","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.031302391Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"client":"0","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.031331759Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"client":"2","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.031874078Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"client":"0","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.031904916Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"client":"0","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.031920196Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.031932855Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"client":"1","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.031946564Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"client":"1","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.031960113Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.031972482Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"client":"2","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.031985782Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"client":"2","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"[0]R flush_numbers[0..12]: [322, 322, 322, 296, 322, 324, 322, 325, 328, 327, 328, 328]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.032009251Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"":"downstairs","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"[0]R generation[0..12]: [189, 189, 189, 176, 189, 191, 189, 192, 193, 192, 193, 193]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.032029769Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"":"downstairs","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.032044589Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"":"downstairs","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"[1]R flush_numbers[0..12]: [322, 322, 322, 296, 322, 324, 322, 325, 328, 327, 328, 328]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.032058538Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"":"downstairs","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"[1]R generation[0..12]: [189, 189, 189, 176, 189, 191, 189, 192, 193, 192, 193, 193]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.032072297Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"":"downstairs","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.032086276Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"":"downstairs","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"[2]R flush_numbers[0..12]: [322, 322, 322, 296, 322, 324, 322, 325, 328, 327, 328, 328]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.032100245Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"":"downstairs","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"[2]R generation[0..12]: [189, 189, 189, 176, 189, 191, 189, 192, 193, 192, 193, 193]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.032114045Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"":"downstairs","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.032127964Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.032144983Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"":"downstairs","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.032158192Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"":"downstairs","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"Next flush: 329","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.032171391Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"":"downstairs","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.032184361Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"":"downstairs","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.03219767Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.032212159Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"client":"0","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.032225678Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"client":"1","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.032238678Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"client":"2","":"downstairs","session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.032251787Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.032264046Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"5a237da8-3039-49df-a446-80485a192238 is now active with session: 85c0f243-8d5f-4978-93e0-a2579e0060b9","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.032276855Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.032288775Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958,"session_id":"85c0f243-8d5f-4978-93e0-a2579e0060b9"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:15.032301014Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 514 len 36864 data: 13 11 11 9 13 14 13 11 8 02/30 Read block 256 len 28672 03/30 Read block 272 len 12288 04/30 Read block 0 len 32768 05/30 Read block 389 len 16384 06/30 Write block 91 len 4096 data: 21 07/30 Read block 329 len 28672 08/30 Read block 411 len 12288 09/30 Read block 290 len 32768 10/30 Write block 426 len 8192 data: 13 15 11/30 Flush 12/30 Write block 298 len 12288 data: 17 16 15 13/30 Read block 591 len 36864 14/30 Write block 245 len 24576 data: 9 10 13 12 13 10 15/30 Read block 434 len 40960 16/30 Write block 557 len 32768 data: 8 8 8 11 12 12 13 11 17/30 Read block 196 len 8192 18/30 Read block 315 len 28672 19/30 Write block 304 len 20480 data: 20 22 16 15 16 20/30 Read block 305 len 4096 21/30 Read block 55 len 32768 22/30 Read block 291 len 24576 23/30 Write block 141 len 8192 data: 14 11 24/30 Flush 25/30 Write block 168 len 40960 data: 15 16 17 19 18 17 14 14 14 11 26/30 Write block 134 len 20480 data: 13 12 12 13 11 27/30 Write block 307 len 4096 data: 16 28/30 Write block 550 len 12288 data: 21 17 15 29/30 Read block 223 len 20480 30/30 Read block 444 len 8192 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 10 Done Done Done false 26 Acked 1025 Write 5 Done Done Done false 27 Acked 1026 Write 1 Done Done Done false 28 Acked 1027 Write 3 Done Done Done false 29 Acked 1028 Read 5 Done Done Done false 30 Acked 1029 Read 2 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 5 6 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:35:15.054936902Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1958} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 322 322 322 F F F 1 020-039 189 189 189 322 322 322 F F F 2 040-059 189 189 189 322 322 322 F F F 3 060-079 176 176 176 296 296 296 F F F 4 080-099 194 194 194 329 329 329 F F F 5 100-119 191 191 191 324 324 324 F F F 6 120-139 189 189 189 322 322 322 T T T 7 140-159 194 194 194 330 330 330 F F F 8 160-179 193 193 193 328 328 328 T T T 9 180-199 192 192 192 327 327 327 F F F 10 200-219 193 193 193 328 328 328 F F F 11 220-239 193 193 193 328 328 328 F F F 12 240-259 194 194 194 330 330 330 F F F 13 260-279 191 191 191 324 324 324 F F F 14 280-299 194 194 194 330 330 330 F F F 15 300-319 194 194 194 330 330 330 T T T 16 320-339 193 193 193 328 328 328 F F F 17 340-359 191 191 191 324 324 324 F F F 18 360-379 192 192 192 327 327 327 F F F 19 380-399 192 192 192 327 327 327 F F F 20 400-419 191 191 191 324 324 324 F F F 21 420-439 194 194 194 329 329 329 F F F 22 440-459 192 192 192 327 327 327 F F F 23 460-479 192 192 192 326 326 326 F F F 24 480-499 192 192 192 326 326 326 F F F 25 500-519 194 194 194 329 329 329 F F F 26 520-539 194 194 194 329 329 329 F F F 27 540-559 194 194 194 330 330 330 T T T 28 560-579 194 194 194 330 330 330 F F F 29 580-599 193 193 193 328 328 328 F F F Max gen: 194, Max flush: 330 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 322 322 322 F F F 1 020-039 189 189 189 322 322 322 F F F 2 040-059 189 189 189 322 322 322 F F F 3 060-079 176 176 176 296 296 296 F F F 4 080-099 194 194 194 329 329 329 F F F 5 100-119 191 191 191 324 324 324 F F F 6 120-139 195 195 195 331 331 331 F F F 7 140-159 194 194 194 330 330 330 F F F 8 160-179 195 195 195 331 331 331 F F F 9 180-199 192 192 192 327 327 327 F F F 10 200-219 193 193 193 328 328 328 F F F 11 220-239 193 193 193 328 328 328 F F F 12 240-259 194 194 194 330 330 330 F F F 13 260-279 191 191 191 324 324 324 F F F 14 280-299 194 194 194 330 330 330 F F F 15 300-319 195 195 195 331 331 331 F F F 16 320-339 193 193 193 328 328 328 F F F 17 340-359 191 191 191 324 324 324 F F F 18 360-379 192 192 192 327 327 327 F F F 19 380-399 192 192 192 327 327 327 F F F 20 400-419 191 191 191 324 324 324 F F F 21 420-439 194 194 194 329 329 329 F F F 22 440-459 192 192 192 327 327 327 F F F 23 460-479 192 192 192 326 326 326 F F F 24 480-499 192 192 192 326 326 326 F F F 25 500-519 194 194 194 329 329 329 F F F 26 520-539 194 194 194 329 329 329 F F F 27 540-559 195 195 195 331 331 331 F F F 28 560-579 194 194 194 330 330 330 F F F 29 580-599 193 193 193 328 328 328 F F F Max gen: 195, Max flush: 331 Begin loop 98 Downstairs to restart: 2 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.392885975Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.39314207Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.393163789Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.393181798Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966} {"msg":"Crucible 2c2a4ddd-e384-4b0a-bdb4-5ecfe11a9113 has session id: cec6d1b4-c5e5-4188-a7bd-4e07480708d7","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.393397366Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"Upstairs opts: Upstairs UUID: 2c2a4ddd-e384-4b0a-bdb4-5ecfe11a9113, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.393438973Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"Crucible stats registered with UUID: 2c2a4ddd-e384-4b0a-bdb4-5ecfe11a9113","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.393455172Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.425062682Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966} {"msg":"2c2a4ddd-e384-4b0a-bdb4-5ecfe11a9113 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.42510253Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.425134068Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"client":"0","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.425180655Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"client":"0","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.425215384Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"client":"1","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.425252371Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"client":"1","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.42528417Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"client":"2","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.425316778Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"client":"2","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.425348746Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.425381124Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.425412772Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"":"io task","client":"1","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:57547)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.42544761Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:46179)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.425480878Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:53009)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.425516607Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"":"io task","client":"1","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.426220587Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"client":"0","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.426247995Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"client":"0","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.426263464Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"client":"1","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.426277233Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"client":"1","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.426294463Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"client":"2","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.426319461Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"client":"2","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.426831602Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"client":"1","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.42686346Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"client":"2","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.426879679Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"client":"0","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.427368562Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"client":"1","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.42739581Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"client":"1","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.427412019Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.427472206Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"client":"0","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.427501374Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"client":"0","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.427516113Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.427528542Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"client":"2","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.427541882Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"client":"2","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"[0]R flush_numbers[0..12]: [322, 322, 322, 296, 329, 324, 331, 330, 331, 327, 328, 328]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.427555741Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"":"downstairs","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"[0]R generation[0..12]: [189, 189, 189, 176, 194, 191, 195, 194, 195, 192, 193, 193]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.42756978Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"":"downstairs","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.427584109Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"":"downstairs","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"[1]R flush_numbers[0..12]: [322, 322, 322, 296, 329, 324, 331, 330, 331, 327, 328, 328]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.427597869Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"":"downstairs","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"[1]R generation[0..12]: [189, 189, 189, 176, 194, 191, 195, 194, 195, 192, 193, 193]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.427611848Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"":"downstairs","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.427629417Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"":"downstairs","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"[2]R flush_numbers[0..12]: [322, 322, 322, 296, 329, 324, 331, 330, 331, 327, 328, 328]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.427643426Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"":"downstairs","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"[2]R generation[0..12]: [189, 189, 189, 176, 194, 191, 195, 194, 195, 192, 193, 193]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.427657235Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"":"downstairs","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.427671314Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"":"downstairs","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.427684064Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"":"downstairs","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.427697243Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"":"downstairs","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"Next flush: 332","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.427709982Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"":"downstairs","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.427722822Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"":"downstairs","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.427736321Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966Wait for a query_work_queue command to finish before sending IO ,"session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.42775124Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"client":"0","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.427765319Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"client":"1","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.427778558Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"client":"2","":"downstairs","session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.427791958Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.427804417Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"2c2a4ddd-e384-4b0a-bdb4-5ecfe11a9113 is now active with session: cec6d1b4-c5e5-4188-a7bd-4e07480708d7","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.427819116Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.427831305Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966,"session_id":"cec6d1b4-c5e5-4188-a7bd-4e07480708d7"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:19.427843665Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 194 len 24576 data: 12 13 14 12 14 11 02/30 Read block 33 len 36864 03/30 Read block 259 len 24576 04/30 Read block 509 len 12288 05/30 Read block 553 len 4096 06/30 Read block 381 len 28672 07/30 Read block 330 len 28672 08/30 Read block 1 len 32768 09/30 Write block 492 len 28672 data: 19 19 20 19 18 18 17 10/30 Read block 549 len 12288 11/30 Write block 285 len 20480 data: 22 17 16 17 19 12/30 Read block 373 len 4096 13/30 Read block 306 len 16384 14/30 Read block 96 len 20480 15/30 Read block 552 len 40960 16/30 Read block 329 len 40960 17/30 Read block 576 len 4096 18/30 Read block 320 len 12288 19/30 Flush 20/30 Write block 278 len 12288 data: 21 20 19 21/30 Read block 145 len 32768 22/30 Read block 114 len 16384 23/30 Read block 305 len 40960 24/30 Read block 188 len 24576 25/30 Write block 153 len 16384 data: 12 12 16 22 26/30 Read block 377 len 16384 27/30 Read block 508 len 12288 28/30 Read block 164 len 28672 29/30 Write block 249 len 28672 data: 14 11 10 13 12 14 16 30/30 Write block 557 len 40960 data: 9 9 9 12 13 13 14 12 14 12 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 20 Acked 1019 Write 3 Done Done Done false 21 Acked 1020 Read 8 Done Done Done false 22 Acked 1021 Read 4 Done Done Done false 23 Acked 1022 Read 10 Done Done Done false 24 Acked 1023 Read 6 Done Done Done false 25 Acked 1024 Write 4 Done Done Done false 26 Acked 1025 Read 4 Done Done Done false 27 Acked 1026 Read 3 Done Done Done false 28 Acked 1027 Read 7 Done Done Done false 29 Acked 1028 Write 7 Sent Sent Sent false 30 Acked 1029 Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:35:19.450863371Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1966} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 322 322 322 F F F 1 020-039 189 189 189 322 322 322 F F F 2 040-059 189 189 189 322 322 322 F F F 3 060-079 176 176 176 296 296 296 F F F 4 080-099 194 194 194 329 329 329 F F F 5 100-119 191 191 191 324 324 324 F F F 6 120-139 195 195 195 331 331 331 F F F 7 140-159 194 194 194 330 330 330 T T T 8 160-179 195 195 195 331 331 331 F F F 9 180-199 196 196 196 332 332 332 F F F 10 200-219 193 193 193 328 328 328 F F F 11 220-239 193 193 193 328 328 328 F F F 12 240-259 194 194 194 330 330 330 T T T 13 260-279 191 191 191 324 324 324 T T T 14 280-299 196 196 196 332 332 332 T T T 15 300-319 195 195 195 331 331 331 F F F 16 320-339 193 193 193 328 328 328 F F F 17 340-359 191 191 191 324 324 324 F F F 18 360-379 192 192 192 327 327 327 F F F 19 380-399 192 192 192 327 327 327 F F F 20 400-419 191 191 191 324 324 324 F F F 21 420-439 194 194 194 329 329 329 F F F 22 440-459 192 192 192 327 327 327 F F F 23 460-479 192 192 192 326 326 326 F F F 24 480-499 196 196 196 332 332 332 F F F 25 500-519 194 194 194 329 329 329 F F F 26 520-539 194 194 194 329 329 329 F F F 27 540-559 195 195 195 331 331 331 T T T 28 560-579 194 194 194 330 330 330 T T T 29 580-599 193 193 193 328 328 328 F F F Max gen: 196, Max flush: 332 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 322 322 322 F F F 1 020-039 189 189 189 322 322 322 F F F 2 040-059 189 189 189 322 322 322 F F F 3 060-079 176 176 176 296 296 296 F F F 4 080-099 194 194 194 329 329 329 F F F 5 100-119 191 191 191 324 324 324 F F F 6 120-139 195 195 195 331 331 331 F F F 7 140-159 197 197 197 333 333 333 F F F 8 160-179 195 195 195 331 331 331 F F F 9 180-199 196 196 196 332 332 332 F F F 10 200-219 193 193 193 328 328 328 F F F 11 220-239 193 193 193 328 328 328 F F F 12 240-259 197 197 197 333 333 333 F F F 13 260-279 197 197 197 333 333 333 F F F 14 280-299 197 197 197 333 333 333 F F F 15 300-319 195 195 195 331 331 331 F F F 16 320-339 193 193 193 328 328 328 F F F 17 340-359 191 191 191 324 324 324 F F F 18 360-379 192 192 192 327 327 327 F F F 19 380-399 192 192 192 327 327 327 F F F 20 400-419 191 191 191 324 324 324 F F F 21 420-439 194 194 194 329 329 329 F F F 22 440-459 192 192 192 327 327 327 F F F 23 460-479 192 192 192 326 326 326 F F F 24 480-499 196 196 196 332 332 332 F F F 25 500-519 194 194 194 329 329 329 F F F 26 520-539 194 194 194 329 329 329 F F F 27 540-559 197 197 197 333 333 333 F F F 28 560-579 197 197 197 333 333 333 F F F 29 580-599 193 193 193 328 328 328 F F F Max gen: 197, Max flush: 333 Begin loop 99 Downstairs to restart: 2 look at: http://127.0.0.1:12810 RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 } {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.810748491Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9d0029dbf9065191a369b89880e9a5f8aa8ee534\",\n git_commit_timestamp: \"2024-09-26T13:46:09.000000000Z\",\n git_branch: \"mkeeter/remove-iostatecount-new\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.811035235Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.811057454Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.811144338Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974} {"msg":"Crucible 26e735f3-b63e-4b1b-ab77-703b9d9d2f07 has session id: 34e84702-4bf6-4480-8bbb-ed1a0255690e","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.811241883Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"Upstairs opts: Upstairs UUID: 26e735f3-b63e-4b1b-ab77-703b9d9d2f07, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.81128709Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"Crucible stats registered with UUID: 26e735f3-b63e-4b1b-ab77-703b9d9d2f07","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.811314989Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.842396809Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974} {"msg":"26e735f3-b63e-4b1b-ab77-703b9d9d2f07 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.842430787Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.842452985Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"client":"0","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.842497523Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"client":"0","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.842532681Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"client":"1","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.842566939Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"client":"1","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.842600117Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"client":"2","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.842630675Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"client":"2","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.842660554Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"":"io task","client":"2","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.842689132Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"":"io task","client":"0","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.842857773Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"ds_connection connected from Ok(127.0.0.1:47499)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.84289752Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"":"io task","client":"0","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"ds_connection connected from Ok(127.0.0.1:61266)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.842919999Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"":"io task","client":"2","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"ds_connection connected from Ok(127.0.0.1:46390)","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.842942158Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.843533684Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"client":"1","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.843561533Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"client":"1","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.843576932Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"client":"2","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.843590851Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"client":"2","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.8436046Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"client":"0","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.843623969Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"client":"0","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.844178208Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"client":"0","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.844217716Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"client":"1","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.844248754Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"client":"2","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.844807632Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"client":"1","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.84484211Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"client":"1","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.844872168Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.844891857Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"client":"0","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.844906086Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"client":"0","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.844919596Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.844931845Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"client":"2","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.844945174Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"client":"2","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"[0]R flush_numbers[0..12]: [322, 322, 322, 296, 329, 324, 331, 333, 331, 332, 328, 328]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.844959223Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"":"downstairs","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"[0]R generation[0..12]: [189, 189, 189, 176, 194, 191, 195, 197, 195, 196, 193, 193]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.844973313Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"":"downstairs","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.844987292Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"":"downstairs","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"[1]R flush_numbers[0..12]: [322, 322, 322, 296, 329, 324, 331, 333, 331, 332, 328, 328]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.845001161Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"":"downstairs","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"[1]R generation[0..12]: [189, 189, 189, 176, 194, 191, 195, 197, 195, 196, 193, 193]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.8450149Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"":"downstairs","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.84502897Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"":"downstairs","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"[2]R flush_numbers[0..12]: [322, 322, 322, 296, 329, 324, 331, 333, 331, 332, 328, 328]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.845042709Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"":"downstairs","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"[2]R generation[0..12]: [189, 189, 189, 176, 194, 191, 195, 197, 195, 196, 193, 193]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.845059678Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"":"downstairs","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.845088056Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.845114215Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"":"downstairs","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.845136744Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"":"downstairs","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"Next flush: 334","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.845161242Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"":"downstairs","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.845181851Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"":"downstairs","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.84520572Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.845226398Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"client":"0","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.845248367Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"client":"1","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 crucible","level":30,"time":"2024-09-26T14:35:23.845278256Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"client":"2","":"downstairs","session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.845301234Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.845314374Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"26e735f3-b63e-4b1b-ab77-703b9d9d2f07 is now active with session: 34e84702-4bf6-4480-8bbb-ed1a0255690e","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.845327233Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.845339272Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974,"session_id":"34e84702-4bf6-4480-8bbb-ed1a0255690e"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-09-26T14:35:23.845351501Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 55 len 20480 data: 12 13 15 13 14 02/30 Write block 443 len 36864 data: 21 22 22 18 17 15 16 15 12 03/30 Write block 59 len 12288 data: 15 15 13 04/30 Write block 192 len 12288 data: 17 13 13 05/30 Read block 465 len 20480 06/30 Read block 153 len 40960 07/30 Write block 362 len 36864 data: 17 17 16 12 11 11 10 10 11 08/30 Read block 471 len 28672 09/30 Read block 552 len 40960 10/30 Write block 587 len 12288 data: 16 15 15 11/30 Read block 207 len 16384 12/30 Read block 95 len 40960 13/30 Read block 185 len 20480 14/30 Flush 15/30 Flush 16/30 Write block 140 len 40960 data: 14 15 12 12 12 11 10 13 8 10 17/30 Write block 118 len 12288 data: 15 17 16 18/30 Write block 590 len 16384 data: 15 13 12 11 19/30 Read block 167 len 8192 20/30 Write block 471 len 36864 data: 13 13 14 14 15 17 17 19 21 21/30 Write block 461 len 8192 data: 11 8 22/30 Read block 24 len 40960 23/30 Read block 18 len 20480 24/30 Read block 261 len 8192 25/30 Flush 26/30 Write block 24 len 24576 data: 15 12 15 15 18 11 27/30 Write block 201 len 16384 data: 12 15 12 16 28/30 Read block 57 len 4096 29/30 Read block 333 len 16384 30/30 Write block 393 len 32768 data: 10 10 11 13 16 17 17 15 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 6 Done Done Done false 27 Acked 1026 Write 4 Done Done Done false 28 Acked 1027 Read 1 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-26T14:35:23.869942099Z","hostname":"w-01J8QBMA0P4PS841BFGZXS21Y3","pid":1974} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 322 322 322 F F F 1 020-039 189 189 189 322 322 322 T T T 2 040-059 198 198 198 334 334 334 F F F 3 060-079 198 198 198 334 334 334 F F F 4 080-099 194 194 194 329 329 329 F F F 5 100-119 198 198 198 336 336 336 F F F 6 120-139 198 198 198 336 336 336 F F F 7 140-159 198 198 198 336 336 336 F F F 8 160-179 195 195 195 331 331 331 F F F 9 180-199 198 198 198 334 334 334 F F F 10 200-219 193 193 193 328 328 328 T T T 11 220-239 193 193 193 328 328 328 F F F 12 240-259 197 197 197 333 333 333 F F F 13 260-279 197 197 197 333 333 333 F F F 14 280-299 197 197 197 333 333 333 F F F 15 300-319 195 195 195 331 331 331 F F F 16 320-339 193 193 193 328 328 328 F F F 17 340-359 191 191 191 324 324 324 F F F 18 360-379 198 198 198 334 334 334 F F F 19 380-399 192 192 192 327 327 327 T T T 20 400-419 191 191 191 324 324 324 T T T 21 420-439 194 194 194 329 329 329 F F F 22 440-459 198 198 198 334 334 334 F F F 23 460-479 198 198 198 336 336 336 F F F 24 480-499 196 196 196 332 332 332 F F F 25 500-519 194 194 194 329 329 329 F F F 26 520-539 194 194 194 329 329 329 F F F 27 540-559 197 197 197 333 333 333 F F F 28 560-579 197 197 197 333 333 333 F F F 29 580-599 198 198 198 336 336 336 F F F Max gen: 198, Max flush: 336 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 322 322 322 F F F 1 020-039 199 199 199 337 337 337 F F F 2 040-059 198 198 198 334 334 334 F F F 3 060-079 198 198 198 334 334 334 F F F 4 080-099 194 194 194 329 329 329 F F F 5 100-119 198 198 198 336 336 336 F F F 6 120-139 198 198 198 336 336 336 F F F 7 140-159 198 198 198 336 336 336 F F F 8 160-179 195 195 195 331 331 331 F F F 9 180-199 198 198 198 334 334 334 F F F 10 200-219 199 199 199 337 337 337 F F F 11 220-239 193 193 193 328 328 328 F F F 12 240-259 197 197 197 333 333 333 F F F 13 260-279 197 197 197 333 333 333 F F F 14 280-299 197 197 197 333 333 333 F F F 15 300-319 195 195 195 331 331 331 F F F 16 320-339 193 193 193 328 328 328 F F F 17 340-359 191 191 191 324 324 324 F F F 18 360-379 198 198 198 334 334 334 F F F 19 380-399 199 199 199 337 337 337 F F F 20 400-419 199 199 199 337 337 337 F F F 21 420-439 194 194 194 329 329 329 F F F 22 440-459 198 198 198 334 334 334 F F F 23 460-479 198 198 198 336 336 336 F F F 24 480-499 196 196 196 332 332 332 F F F 25 500-519 194 194 194 329 329 329 F F F 26 520-539 194 194 194 329 329 329 F F F 27 540-559 197 197 197 333 333 333 F F F 28 560-579 197 197 197 333 333 333 F F F 29 580-599 198 198 198 336 336 336 F F F Max gen: 199, Max flush: 337 7:26 Test duration Test completed real 7:25.417890769 user 56.777916127 sys 18.379212943 trap 0.070473965 tflt 0.038174385 dflt 0.091813556 kflt 0.006187036 lock 11:53:27.034864784 slp 36:09.716980194 lat 5.579654679 stop 0.059720160